From 303bbc42f4322448998f3fde202574335d1190e6 Mon Sep 17 00:00:00 2001
From: Alex Browne <stephenalexbrowne@gmail.com>
Date: Wed, 14 Nov 2018 15:58:36 -0800
Subject: Change some column types from varchar to numeric

---
 .../src/entities/exchange_cancel_up_to_event.ts    |  7 ++++--
 .../pipeline/src/entities/exchange_fill_event.ts   | 18 ++++++++-------
 packages/pipeline/src/entities/sra_order.ts        | 26 ++++++++++++----------
 packages/pipeline/src/ormconfig.ts                 |  2 +-
 packages/pipeline/src/parsers/events/index.ts      | 10 ++++-----
 packages/pipeline/src/parsers/sra_orders/index.ts  | 12 +++++-----
 .../pipeline/src/scripts/pull_missing_events.ts    |  2 +-
 packages/pipeline/src/utils/index.ts               | 15 +++++++++++++
 8 files changed, 57 insertions(+), 35 deletions(-)

(limited to 'packages/pipeline/src')

diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
index 7306a1a87..752631b85 100644
--- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -1,5 +1,8 @@
+import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
+import { bigNumberTransformer } from '../utils';
+
 @Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' })
 export class ExchangeCancelUpToEvent {
     @PrimaryColumn({ name: 'contract_address' })
@@ -17,7 +20,7 @@ export class ExchangeCancelUpToEvent {
     public makerAddress!: string;
     @Column({ name: 'sender_address' })
     public senderAddress!: string;
-    @Column({ name: 'order_epoch' })
-    public orderEpoch!: string;
+    @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer })
+    public orderEpoch!: BigNumber;
     // TODO(albrow): Include topics?
 }
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
index 6202e558b..aa082436b 100644
--- a/packages/pipeline/src/entities/exchange_fill_event.ts
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
 
 @Entity({ name: 'exchange_fill_events', schema: 'raw' })
 export class ExchangeFillEvent {
@@ -24,14 +26,14 @@ export class ExchangeFillEvent {
     public feeRecipientAddress!: string;
     @Column({ name: 'sender_address' })
     public senderAddress!: string;
-    @Column({ name: 'maker_asset_filled_amount' })
-    public makerAssetFilledAmount!: string;
-    @Column({ name: 'taker_asset_filled_amount' })
-    public takerAssetFilledAmount!: string;
-    @Column({ name: 'maker_fee_paid' })
-    public makerFeePaid!: string;
-    @Column({ name: 'taker_fee_paid' })
-    public takerFeePaid!: string;
+    @Column({ name: 'maker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerAssetFilledAmount!: BigNumber;
+    @Column({ name: 'taker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerAssetFilledAmount!: BigNumber;
+    @Column({ name: 'maker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+    public makerFeePaid!: BigNumber;
+    @Column({ name: 'taker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+    public takerFeePaid!: BigNumber;
     @Column({ name: 'order_hash' })
     public orderHash!: string;
 
diff --git a/packages/pipeline/src/entities/sra_order.ts b/packages/pipeline/src/entities/sra_order.ts
index 4b7f652d3..9c730a0bb 100644
--- a/packages/pipeline/src/entities/sra_order.ts
+++ b/packages/pipeline/src/entities/sra_order.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
 
 @Entity({ name: 'sra_orders', schema: 'raw' })
 export class SraOrder {
@@ -19,18 +21,18 @@ export class SraOrder {
     public feeRecipientAddress!: string;
     @Column({ name: 'sender_address' })
     public senderAddress!: string;
-    @Column({ name: 'maker_asset_amount' })
-    public makerAssetAmount!: string;
-    @Column({ name: 'taker_asset_amount' })
-    public takerAssetAmount!: string;
-    @Column({ name: 'maker_fee' })
-    public makerFee!: string;
-    @Column({ name: 'taker_fee' })
-    public takerFee!: string;
-    @Column({ name: 'expiration_time_seconds' })
-    public expirationTimeSeconds!: string;
-    @Column({ name: 'salt' })
-    public salt!: string;
+    @Column({ name: 'maker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerAssetAmount!: BigNumber;
+    @Column({ name: 'taker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerAssetAmount!: BigNumber;
+    @Column({ name: 'maker_fee', type: 'numeric', transformer: bigNumberTransformer })
+    public makerFee!: BigNumber;
+    @Column({ name: 'taker_fee', type: 'numeric', transformer: bigNumberTransformer })
+    public takerFee!: BigNumber;
+    @Column({ name: 'expiration_time_seconds', type: 'numeric', transformer: bigNumberTransformer })
+    public expirationTimeSeconds!: BigNumber;
+    @Column({ name: 'salt', type: 'numeric', transformer: bigNumberTransformer })
+    public salt!: BigNumber;
     @Column({ name: 'signature' })
     public signature!: string;
 
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 2f5f7df33..0e489e560 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -26,7 +26,7 @@ const config: ConnectionOptions = {
     type: 'postgres',
     url: process.env.ZEROEX_DATA_PIPELINE_DB_URL,
     synchronize: false,
-    logging: false,
+    logging: ['error', 'warn'],
     entities,
     migrations: ['./lib/migrations/**/*.js'],
 };
diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts
index d56dc7080..27e192d7b 100644
--- a/packages/pipeline/src/parsers/events/index.ts
+++ b/packages/pipeline/src/parsers/events/index.ts
@@ -65,10 +65,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
     exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
     exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
     exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
-    exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString();
-    exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString();
-    exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid.toString();
-    exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid.toString();
+    exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount;
+    exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount;
+    exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid;
+    exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid;
     exchangeFillEvent.orderHash = eventLog.args.orderHash;
     exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData;
     exchangeFillEvent.makerAssetType = makerAssetType;
@@ -139,6 +139,6 @@ export function _convertToExchangeCancelUpToEvent(
     exchangeCancelUpToEvent.rawData = eventLog.data as string;
     exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress.toString();
     exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress.toString();
-    exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch.toString();
+    exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch;
     return exchangeCancelUpToEvent;
 }
diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts
index 800521955..ef8901e40 100644
--- a/packages/pipeline/src/parsers/sra_orders/index.ts
+++ b/packages/pipeline/src/parsers/sra_orders/index.ts
@@ -34,12 +34,12 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder {
     sraOrder.takerAddress = apiOrder.order.takerAddress;
     sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress;
     sraOrder.senderAddress = apiOrder.order.senderAddress;
-    sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount.toString();
-    sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount.toString();
-    sraOrder.makerFee = apiOrder.order.makerFee.toString();
-    sraOrder.takerFee = apiOrder.order.takerFee.toString();
-    sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds.toString();
-    sraOrder.salt = apiOrder.order.salt.toString();
+    sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount;
+    sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount;
+    sraOrder.makerFee = apiOrder.order.makerFee;
+    sraOrder.takerFee = apiOrder.order.takerFee;
+    sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds;
+    sraOrder.salt = apiOrder.order.salt;
     sraOrder.signature = apiOrder.order.signature;
 
     sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 0af999a77..1693bb59a 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -13,7 +13,7 @@ import { handleError } from '../utils';
 
 const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
 const START_BLOCK_OFFSET = 1000; // Number of blocks before the last known block to consider when updating fill events.
-const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
+const BATCH_SAVE_SIZE = 10000; // Number of events to save at once.
 
 let connection: Connection;
 
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index a083cd8c9..78aa89374 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -1,4 +1,5 @@
 import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
 
 /**
  * If the given BigNumber is not null, returns the string representation of that
@@ -34,3 +35,17 @@ export function handleError(e: any): void {
     }
     process.exit(1);
 }
+
+class BigNumberTransformer implements ValueTransformer {
+    // tslint:disable-next-line:prefer-function-over-method
+    public to(value: BigNumber): string {
+        return value.toString();
+    }
+
+    // tslint:disable-next-line:prefer-function-over-method
+    public from(value: string): BigNumber {
+        return new BigNumber(value);
+    }
+}
+
+export const bigNumberTransformer = new BigNumberTransformer();
-- 
cgit