From 00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 4 Dec 2018 20:04:08 -0800 Subject: Address PR feedback --- packages/pipeline/README.md | 21 +++++++------- ...9179-ConvertTokenMetadataDecimalsToBigNumber.ts | 17 +++++++++++ packages/pipeline/package.json | 7 +---- .../contract-wrappers/exchange_events.ts | 3 +- .../pipeline/src/entities/exchange_cancel_event.ts | 2 -- .../src/entities/exchange_cancel_up_to_event.ts | 2 -- .../pipeline/src/entities/exchange_fill_event.ts | 2 -- packages/pipeline/src/entities/relayer.ts | 3 -- packages/pipeline/src/entities/token_metadata.ts | 9 +++--- packages/pipeline/src/parsers/events/index.ts | 13 ++++----- .../pipeline/src/parsers/relayer_registry/index.ts | 13 +++++---- .../pipeline/src/parsers/token_metadata/index.ts | 6 ++-- packages/pipeline/src/parsers/web3/index.ts | 1 - .../pipeline/src/scripts/pull_missing_blocks.ts | 7 ++--- .../pipeline/src/scripts/pull_missing_events.ts | 5 ++-- .../src/scripts/pull_radar_relay_orders.ts | 33 +++++++++++++--------- .../pipeline/src/scripts/pull_trusted_tokens.ts | 14 ++++----- packages/pipeline/src/utils/constants.ts | 3 ++ packages/pipeline/src/utils/index.ts | 1 + .../pipeline/src/utils/transformers/big_number.ts | 8 +++--- .../pipeline/test/entities/token_metadata_test.ts | 7 +++-- 21 files changed, 96 insertions(+), 81 deletions(-) create mode 100644 packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts create mode 100644 packages/pipeline/src/utils/constants.ts (limited to 'packages/pipeline') diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md index fb563b14c..0f4abd935 100644 --- a/packages/pipeline/README.md +++ b/packages/pipeline/README.md @@ -42,7 +42,8 @@ Revert the most recent migration (CAUTION: may result in data loss!): `yarn migr There are several test scripts in **package.json**. You can run all the tests with `yarn test:all` or run certain tests seprately by following the -instructions below. Some tests may not work out of the box on certain platforms. +instructions below. Some tests may not work out of the box on certain platforms +or operating systems (see the "Database tests" section below). ### Unit tests @@ -71,8 +72,8 @@ Postgres is via Docker. Depending on your platform, you may need to prepend docker run --rm -d -p 5432:5432 --name pipeline_postgres postgres:11-alpine ``` -This will start a Postgres server with the default username and database name. -You should set the environment variable as follows: +This will start a Postgres server with the default username and database name +(`postgres` and `postgres`). You should set the environment variable as follows: ``` export ZEROEX_DATA_PIPELINE_DB_URL=postgresql://postgres@localhost/postgres @@ -149,17 +150,17 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid #### Additional guidelines and tips: -* Table names should be plural and separated by underscores (e.g., +- Table names should be plural and separated by underscores (e.g., `exchange_fill_events`). -* Any table which contains data which comes directly from a third-party source +- Any table which contains data which comes directly from a third-party source should be namespaced in the `raw` PostgreSQL schema. -* Column names in the database should be separated by underscores (e.g., +- Column names in the database should be separated by underscores (e.g., `maker_asset_type`). -* Field names in entity classes (like any other fields in TypeScript) should +- Field names in entity classes (like any other fields in TypeScript) should be camel-cased (e.g., `makerAssetType`). -* All timestamps should be stored as milliseconds since the Unix Epoch. -* Use the `BigNumber` type for TypeScript code which deals with 256-bit +- All timestamps should be stored as milliseconds since the Unix Epoch. +- Use the `BigNumber` type for TypeScript code which deals with 256-bit numbers from smart contracts or for any case where we are dealing with large floating point numbers. -* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a +- [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a helpful resource. diff --git a/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts new file mode 100644 index 000000000..351bc7eb8 --- /dev/null +++ b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ConvertTokenMetadataDecimalsToBigNumber1543980079179 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE raw.token_metadata + ALTER COLUMN decimals TYPE numeric USING decimals::numeric;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE raw.token_metadata + ALTER COLUMN decimals TYPE numeric USING decimals::integer;`, + ); + } +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 6af81c048..0539618d4 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -1,6 +1,6 @@ { "name": "@0x/pipeline", - "version": "0.0.1", + "version": "1.0.0", "private": true, "description": "Data pipeline for offline analysis", "scripts": { @@ -21,11 +21,6 @@ "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" }, - "config": { - "postpublish": { - "assets": [] - } - }, "repository": { "type": "git", "url": "https://github.com/0xProject/0x-monorepo" diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts index 5721c0ea3..1717eb8b3 100644 --- a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts +++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts @@ -11,9 +11,10 @@ import { Web3ProviderEngine } from '@0x/subproviders'; import { Web3Wrapper } from '@0x/web3-wrapper'; import { LogWithDecodedArgs } from 'ethereum-types'; +import { EXCHANGE_START_BLOCK } from '../../utils'; + const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default toBlock. const NUM_BLOCKS_PER_QUERY = 20000; // Number of blocks to query for events at a time. -const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. export class ExchangeEventsSource { private readonly _exchangeWrapper: ExchangeWrapper; diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts index 2fcc17df6..38f99c903 100644 --- a/packages/pipeline/src/entities/exchange_cancel_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_event.ts @@ -48,6 +48,4 @@ export class ExchangeCancelEvent { public takerTokenAddress!: string; @Column({ nullable: true, type: String, name: 'taker_token_id' }) public takerTokenId!: string | null; - - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts index 60ead324f..27580305e 100644 --- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts @@ -12,7 +12,6 @@ export class ExchangeCancelUpToEvent { @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer }) public blockNumber!: number; - // TODO(albrow): Include transaction hash @Column({ name: 'raw_data' }) public rawData!: string; @@ -24,5 +23,4 @@ export class ExchangeCancelUpToEvent { public senderAddress!: string; @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer }) public orderEpoch!: BigNumber; - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts index bbf0abf58..9b7727615 100644 --- a/packages/pipeline/src/entities/exchange_fill_event.ts +++ b/packages/pipeline/src/entities/exchange_fill_event.ts @@ -57,6 +57,4 @@ export class ExchangeFillEvent { public takerTokenAddress!: string; @Column({ nullable: true, type: String, name: 'taker_token_id' }) public takerTokenId!: string | null; - - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/relayer.ts b/packages/pipeline/src/entities/relayer.ts index b3a856fd8..5af8578b4 100644 --- a/packages/pipeline/src/entities/relayer.ts +++ b/packages/pipeline/src/entities/relayer.ts @@ -14,9 +14,6 @@ export class Relayer { @Column({ name: 'app_url', type: 'varchar', nullable: true }) public appUrl!: string | null; - // TODO(albrow): Add exchange contract or protocol version? - // TODO(albrow): Add network ids for addresses? - @Column({ name: 'fee_recipient_addresses', type: 'varchar', array: true }) public feeRecipientAddresses!: string[]; @Column({ name: 'taker_addresses', type: 'varchar', array: true }) diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts index ca1e57937..911b53972 100644 --- a/packages/pipeline/src/entities/token_metadata.ts +++ b/packages/pipeline/src/entities/token_metadata.ts @@ -1,5 +1,8 @@ +import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; +import { bigNumberTransformer } from '../utils/transformers'; + @Entity({ name: 'token_metadata', schema: 'raw' }) export class TokenMetadata { @PrimaryColumn({ type: 'varchar', nullable: false }) @@ -8,10 +11,8 @@ export class TokenMetadata { @PrimaryColumn({ type: 'varchar', nullable: false }) public authority!: string; - // TODO(albrow): Convert decimals field to type BigNumber/numeric because it - // comes from a 256-bit integer in a smart contract. - @Column({ type: 'integer', nullable: true }) - public decimals!: number | null; + @Column({ type: 'numeric', transformer: bigNumberTransformer, nullable: true }) + public decimals!: BigNumber | null; @Column({ type: 'varchar', nullable: true }) public symbol!: string | null; diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts index d42d1c57a..e18106c75 100644 --- a/packages/pipeline/src/parsers/events/index.ts +++ b/packages/pipeline/src/parsers/events/index.ts @@ -49,8 +49,8 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ - rpcUrl: `https://mainnet.infura.io/${process.env.INFURA_API_KEY}`, + rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`, }); const web3Source = new Web3Source(provider); await getAllMissingBlocks(web3Source); diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts index c7e07c540..80abbb8b0 100644 --- a/packages/pipeline/src/scripts/pull_missing_events.ts +++ b/packages/pipeline/src/scripts/pull_missing_events.ts @@ -8,9 +8,8 @@ import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events'; -import { handleError } from '../utils'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; -const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. const BATCH_SAVE_SIZE = 1000; // Number of events to save at once. @@ -19,7 +18,7 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ - rpcUrl: 'https://mainnet.infura.io', + rpcUrl: INFURA_ROOT_URL, }); const eventsSource = new ExchangeEventsSource(provider, 1); await getFillEventsAsync(eventsSource); diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts index b3a4d887e..bbbef9b47 100644 --- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts +++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts @@ -16,11 +16,11 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); - await getOrderbook(); + await getOrderbookAsync(); process.exit(0); })().catch(handleError); -async function getOrderbook(): Promise { +async function getOrderbookAsync(): Promise { console.log('Getting all orders...'); const connectClient = new HttpClient(RADAR_RELAY_URL); const rawOrders = await connectClient.getOrdersAsync({ @@ -29,17 +29,22 @@ async function getOrderbook(): Promise { console.log(`Got ${rawOrders.records.length} orders.`); console.log('Parsing orders...'); // Parse the sra orders, then add source url to each. - const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders); + const orders = R.pipe( + parseSraOrders, + R.map(setSourceUrl(RADAR_RELAY_URL)), + )(rawOrders); // Save all the orders and update the observed time stamps in a single // transaction. console.log('Saving orders and updating timestamps...'); - await connection.transaction(async (manager: EntityManager): Promise => { - for (const order of orders) { - await manager.save(SraOrder, order); - const observedTimestamp = createObservedTimestampForOrder(order); - await manager.save(observedTimestamp); - } - }); + await connection.transaction( + async (manager: EntityManager): Promise => { + for (const order of orders) { + await manager.save(SraOrder, order); + const observedTimestamp = createObservedTimestampForOrder(order); + await manager.save(observedTimestamp); + } + }, + ); } const sourceUrlProp = R.lensProp('sourceUrl'); @@ -48,6 +53,8 @@ const sourceUrlProp = R.lensProp('sourceUrl'); * Sets the source url for a single order. Returns a new order instead of * mutating the given one. */ -const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => { - return R.set(sourceUrlProp, sourceURL, order); -}); +const setSourceUrl = R.curry( + (sourceURL: string, order: SraOrder): SraOrder => { + return R.set(sourceUrlProp, sourceURL, order); + }, +); diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts index b3a4466bb..1befc4437 100644 --- a/packages/pipeline/src/scripts/pull_trusted_tokens.ts +++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts @@ -8,7 +8,7 @@ import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers import { handleError } from '../utils'; const METAMASK_TRUSTED_TOKENS_URL = - 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json'; + 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/d45916c533116510cc8e9e048a8b5fc3732a6b6d/contract-map.json'; const ZEROEX_TRUSTED_TOKENS_URL = 'https://website-api.0xproject.com/tokens'; @@ -22,7 +22,7 @@ let connection: Connection; })().catch(handleError); async function getMetamaskTrustedTokens(): Promise { - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Getting latest metamask trusted tokens list ...'); const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource>( @@ -30,23 +30,23 @@ async function getMetamaskTrustedTokens(): Promise { ); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseMetamaskTrustedTokens(resp); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Saving metamask trusted tokens list'); await trustedTokensRepository.save(trustedTokens); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Done saving metamask trusted tokens.'); } async function getZeroExTrustedTokens(): Promise { - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Getting latest 0x trusted tokens list ...'); const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource(ZEROEX_TRUSTED_TOKENS_URL); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseZeroExTrustedTokens(resp); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Saving metamask trusted tokens list'); await trustedTokensRepository.save(trustedTokens); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Done saving metamask trusted tokens.'); } diff --git a/packages/pipeline/src/utils/constants.ts b/packages/pipeline/src/utils/constants.ts new file mode 100644 index 000000000..56f3e82d8 --- /dev/null +++ b/packages/pipeline/src/utils/constants.ts @@ -0,0 +1,3 @@ +// Block number when the Exchange contract was deployed to mainnet. +export const EXCHANGE_START_BLOCK = 6271590; +export const INFURA_ROOT_URL = 'https://mainnet.infura.io'; diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts index 718ea6133..2096a0a39 100644 --- a/packages/pipeline/src/utils/index.ts +++ b/packages/pipeline/src/utils/index.ts @@ -1,5 +1,6 @@ import { BigNumber } from '@0x/utils'; export * from './transformers'; +export * from './constants'; /** * If the given BigNumber is not null, returns the string representation of that diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts index a0471a5e3..5f2e4d565 100644 --- a/packages/pipeline/src/utils/transformers/big_number.ts +++ b/packages/pipeline/src/utils/transformers/big_number.ts @@ -3,13 +3,13 @@ import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer'; export class BigNumberTransformer implements ValueTransformer { // tslint:disable-next-line:prefer-function-over-method - public to(value: BigNumber): string { - return value.toString(); + public to(value: BigNumber | null): string | null { + return value === null ? null : value.toString(); } // tslint:disable-next-line:prefer-function-over-method - public from(value: string): BigNumber { - return new BigNumber(value); + public from(value: string | null): BigNumber | null { + return value === null ? null : new BigNumber(value); } } diff --git a/packages/pipeline/test/entities/token_metadata_test.ts b/packages/pipeline/test/entities/token_metadata_test.ts index 805b4b0f7..48e656644 100644 --- a/packages/pipeline/test/entities/token_metadata_test.ts +++ b/packages/pipeline/test/entities/token_metadata_test.ts @@ -1,3 +1,4 @@ +import { BigNumber } from '@0x/utils'; import 'mocha'; import 'reflect-metadata'; @@ -9,15 +10,15 @@ import { testSaveAndFindEntityAsync } from './util'; chaiSetup.configure(); -const metadataWithoutNullFields = { +const metadataWithoutNullFields: TokenMetadata = { address: '0xe41d2489571d322189246dafa5ebde1f4699f498', authority: 'https://website-api.0xproject.com/tokens', - decimals: 18, + decimals: new BigNumber(18), symbol: 'ZRX', name: '0x', }; -const metadataWithNullFields = { +const metadataWithNullFields: TokenMetadata = { address: '0xe41d2489571d322189246dafa5ebde1f4699f499', authority: 'https://website-api.0xproject.com/tokens', decimals: null, -- cgit