From 303bbc42f4322448998f3fde202574335d1190e6 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 14 Nov 2018 15:58:36 -0800 Subject: Change some column types from varchar to numeric --- .../1542234704666-ConvertBigNumberToNumeric.ts | 53 ++++++++++++++++++++++ .../src/entities/exchange_cancel_up_to_event.ts | 7 ++- .../pipeline/src/entities/exchange_fill_event.ts | 18 ++++---- packages/pipeline/src/entities/sra_order.ts | 26 ++++++----- packages/pipeline/src/ormconfig.ts | 2 +- packages/pipeline/src/parsers/events/index.ts | 10 ++-- packages/pipeline/src/parsers/sra_orders/index.ts | 12 ++--- .../pipeline/src/scripts/pull_missing_events.ts | 2 +- packages/pipeline/src/utils/index.ts | 15 ++++++ .../pipeline/test/parsers/events/index_test.ts | 10 ++-- .../pipeline/test/parsers/sra_orders/index_test.ts | 12 ++--- 11 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts diff --git a/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts new file mode 100644 index 000000000..5200ef7cc --- /dev/null +++ b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts @@ -0,0 +1,53 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ConvertBigNumberToNumeric1542234704666 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE raw.exchange_fill_events + ALTER COLUMN maker_asset_filled_amount TYPE numeric USING maker_asset_filled_amount::numeric, + ALTER COLUMN taker_asset_filled_amount TYPE numeric USING taker_asset_filled_amount::numeric, + ALTER COLUMN maker_fee_paid TYPE numeric USING maker_fee_paid::numeric, + ALTER COLUMN taker_fee_paid TYPE numeric USING taker_fee_paid::numeric;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.exchange_cancel_up_to_events + ALTER COLUMN order_epoch TYPE numeric USING order_epoch::numeric;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.sra_orders + ALTER COLUMN maker_asset_amount TYPE numeric USING maker_asset_amount::numeric, + ALTER COLUMN taker_asset_amount TYPE numeric USING taker_asset_amount::numeric, + ALTER COLUMN maker_fee TYPE numeric USING maker_fee::numeric, + ALTER COLUMN taker_fee TYPE numeric USING taker_fee::numeric, + ALTER COLUMN expiration_time_seconds TYPE numeric USING expiration_time_seconds::numeric, + ALTER COLUMN salt TYPE numeric USING salt::numeric;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE raw.sra_orders + ALTER COLUMN maker_asset_amount TYPE varchar USING maker_asset_amount::varchar, + ALTER COLUMN taker_asset_amount TYPE varchar USING taker_asset_amount::varchar, + ALTER COLUMN maker_fee TYPE varchar USING maker_fee::varchar, + ALTER COLUMN taker_fee TYPE varchar USING taker_fee::varchar, + ALTER COLUMN expiration_time_seconds TYPE varchar USING expiration_time_seconds::varchar, + ALTER COLUMN salt TYPE varchar USING salt::varchar;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.exchange_cancel_up_to_events + ALTER COLUMN order_epoch TYPE varchar USING order_epoch::varchar;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.exchange_fill_events + ALTER COLUMN maker_asset_filled_amount TYPE varchar USING maker_asset_filled_amount::varchar, + ALTER COLUMN taker_asset_filled_amount TYPE varchar USING taker_asset_filled_amount::varchar, + ALTER COLUMN maker_fee_paid TYPE varchar USING maker_fee_paid::varchar, + ALTER COLUMN taker_fee_paid TYPE varchar USING taker_fee_paid::varchar;`, + ); + } +} diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts index 7306a1a87..752631b85 100644 --- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts @@ -1,5 +1,8 @@ +import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; +import { bigNumberTransformer } from '../utils'; + @Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' }) export class ExchangeCancelUpToEvent { @PrimaryColumn({ name: 'contract_address' }) @@ -17,7 +20,7 @@ export class ExchangeCancelUpToEvent { public makerAddress!: string; @Column({ name: 'sender_address' }) public senderAddress!: string; - @Column({ name: 'order_epoch' }) - public orderEpoch!: string; + @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer }) + public orderEpoch!: BigNumber; // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts index 6202e558b..aa082436b 100644 --- a/packages/pipeline/src/entities/exchange_fill_event.ts +++ b/packages/pipeline/src/entities/exchange_fill_event.ts @@ -1,6 +1,8 @@ +import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; import { AssetType } from '../types'; +import { bigNumberTransformer } from '../utils'; @Entity({ name: 'exchange_fill_events', schema: 'raw' }) export class ExchangeFillEvent { @@ -24,14 +26,14 @@ export class ExchangeFillEvent { public feeRecipientAddress!: string; @Column({ name: 'sender_address' }) public senderAddress!: string; - @Column({ name: 'maker_asset_filled_amount' }) - public makerAssetFilledAmount!: string; - @Column({ name: 'taker_asset_filled_amount' }) - public takerAssetFilledAmount!: string; - @Column({ name: 'maker_fee_paid' }) - public makerFeePaid!: string; - @Column({ name: 'taker_fee_paid' }) - public takerFeePaid!: string; + @Column({ name: 'maker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer }) + public makerAssetFilledAmount!: BigNumber; + @Column({ name: 'taker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer }) + public takerAssetFilledAmount!: BigNumber; + @Column({ name: 'maker_fee_paid', type: 'numeric', transformer: bigNumberTransformer }) + public makerFeePaid!: BigNumber; + @Column({ name: 'taker_fee_paid', type: 'numeric', transformer: bigNumberTransformer }) + public takerFeePaid!: BigNumber; @Column({ name: 'order_hash' }) public orderHash!: string; diff --git a/packages/pipeline/src/entities/sra_order.ts b/packages/pipeline/src/entities/sra_order.ts index 4b7f652d3..9c730a0bb 100644 --- a/packages/pipeline/src/entities/sra_order.ts +++ b/packages/pipeline/src/entities/sra_order.ts @@ -1,6 +1,8 @@ +import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; import { AssetType } from '../types'; +import { bigNumberTransformer } from '../utils'; @Entity({ name: 'sra_orders', schema: 'raw' }) export class SraOrder { @@ -19,18 +21,18 @@ export class SraOrder { public feeRecipientAddress!: string; @Column({ name: 'sender_address' }) public senderAddress!: string; - @Column({ name: 'maker_asset_amount' }) - public makerAssetAmount!: string; - @Column({ name: 'taker_asset_amount' }) - public takerAssetAmount!: string; - @Column({ name: 'maker_fee' }) - public makerFee!: string; - @Column({ name: 'taker_fee' }) - public takerFee!: string; - @Column({ name: 'expiration_time_seconds' }) - public expirationTimeSeconds!: string; - @Column({ name: 'salt' }) - public salt!: string; + @Column({ name: 'maker_asset_amount', type: 'numeric', transformer: bigNumberTransformer }) + public makerAssetAmount!: BigNumber; + @Column({ name: 'taker_asset_amount', type: 'numeric', transformer: bigNumberTransformer }) + public takerAssetAmount!: BigNumber; + @Column({ name: 'maker_fee', type: 'numeric', transformer: bigNumberTransformer }) + public makerFee!: BigNumber; + @Column({ name: 'taker_fee', type: 'numeric', transformer: bigNumberTransformer }) + public takerFee!: BigNumber; + @Column({ name: 'expiration_time_seconds', type: 'numeric', transformer: bigNumberTransformer }) + public expirationTimeSeconds!: BigNumber; + @Column({ name: 'salt', type: 'numeric', transformer: bigNumberTransformer }) + public salt!: BigNumber; @Column({ name: 'signature' }) public signature!: string; diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 2f5f7df33..0e489e560 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -26,7 +26,7 @@ const config: ConnectionOptions = { type: 'postgres', url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, synchronize: false, - logging: false, + logging: ['error', 'warn'], entities, migrations: ['./lib/migrations/**/*.js'], }; diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts index d56dc7080..27e192d7b 100644 --- a/packages/pipeline/src/parsers/events/index.ts +++ b/packages/pipeline/src/parsers/events/index.ts @@ -65,10 +65,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs { makerAssetFilledAmount: new BigNumber('10000000000000000'), takerAssetFilledAmount: new BigNumber('100000000000000000'), makerFeePaid: new BigNumber('0'), - takerFeePaid: new BigNumber('0'), + takerFeePaid: new BigNumber('12345'), orderHash: '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a', makerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', takerAssetData: '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498', @@ -56,10 +56,10 @@ describe('exchange_events', () => { expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd'; expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; - expected.makerAssetFilledAmount = '10000000000000000'; - expected.takerAssetFilledAmount = '100000000000000000'; - expected.makerFeePaid = '0'; - expected.takerFeePaid = '0'; + expected.makerAssetFilledAmount = new BigNumber('10000000000000000'); + expected.takerAssetFilledAmount = new BigNumber('100000000000000000'); + expected.makerFeePaid = new BigNumber('0'); + expected.takerFeePaid = new BigNumber('12345'); expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a'; expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; expected.makerAssetType = 'erc20'; diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts index 534d84ab3..ee2842ef3 100644 --- a/packages/pipeline/test/parsers/sra_orders/index_test.ts +++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts @@ -41,12 +41,12 @@ describe('sra_orders', () => { expected.takerAddress = '0x0000000000000000000000000000000000000000'; expected.feeRecipientAddress = '0xa258b39954cef5cb142fd567a46cddb31a670124'; expected.senderAddress = '0x0000000000000000000000000000000000000000'; - expected.makerAssetAmount = '1619310371000000000'; - expected.takerAssetAmount = '8178335207070707070707'; - expected.makerFee = '0'; - expected.takerFee = '0'; - expected.expirationTimeSeconds = '1538529488'; - expected.salt = '1537924688891'; + expected.makerAssetAmount = new BigNumber('1619310371000000000'); + expected.takerAssetAmount = new BigNumber('8178335207070707070707'); + expected.makerFee = new BigNumber('0'); + expected.takerFee = new BigNumber('0'); + expected.expirationTimeSeconds = new BigNumber('1538529488'); + expected.salt = new BigNumber('1537924688891'); expected.signature = '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03'; expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; -- cgit