Update to use ContractWrappers + Infura instead of Etherscan

6 files changed, 71 insertions, 135 deletions

diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
new file mode 100644
index 000000000..77217c601
--- /dev/null
+++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
@@ -0,0 +1,51 @@
+import { ContractWrappers, ExchangeEvents, ExchangeFillEventArgs, ExchangeWrapper } from '@0xproject/contract-wrappers';
+import { Web3ProviderEngine } from '@0xproject/subproviders';
+import { Web3Wrapper } from '@0xproject/web3-wrapper';
+import { LogWithDecodedArgs } from 'ethereum-types';
+
+const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default toBlock.
+const NUM_BLOCKS_PER_QUERY = 100000; // Number of blocks to query for events at a time.
+const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
+
+export class ExchangeEventsSource {
+    private _exchangeWrapper: ExchangeWrapper;
+    private _web3Wrapper: Web3Wrapper;
+    constructor(provider: Web3ProviderEngine, networkId: number) {
+        this._web3Wrapper = new Web3Wrapper(provider);
+        const contractWrappers = new ContractWrappers(provider, { networkId });
+        this._exchangeWrapper = contractWrappers.exchange;
+    }
+
+    // TODO(albrow): Get Cancel and CancelUpTo events.
+
+    public async getFillEventsAsync(
+        fromBlock: number = EXCHANGE_START_BLOCK,
+        toBlock?: number,
+    ): Promise<Array<LogWithDecodedArgs<ExchangeFillEventArgs>>> {
+        const calculatedToBlock =
+            toBlock === undefined
+                ? (await this._web3Wrapper.getBlockNumberAsync()) - BLOCK_FINALITY_THRESHOLD
+                : toBlock;
+        let events: Array<LogWithDecodedArgs<ExchangeFillEventArgs>> = [];
+        for (let currFromBlock = fromBlock; currFromBlock <= calculatedToBlock; currFromBlock += NUM_BLOCKS_PER_QUERY) {
+            events = events.concat(
+                await this._getFillEventsForRangeAsync(currFromBlock, currFromBlock + NUM_BLOCKS_PER_QUERY - 1),
+            );
+        }
+        return events;
+    }
+
+    private async _getFillEventsForRangeAsync(
+        fromBlock: number,
+        toBlock: number,
+    ): Promise<Array<LogWithDecodedArgs<ExchangeFillEventArgs>>> {
+        return this._exchangeWrapper.getLogsAsync<ExchangeFillEventArgs>(
+            ExchangeEvents.Fill,
+            {
+                fromBlock,
+                toBlock,
+            },
+            {},
+        );
+    }
+}
diff --git a/packages/pipeline/src/data_sources/etherscan/index.ts b/packages/pipeline/src/data_sources/etherscan/index.ts
deleted file mode 100644
index 044fff02e..000000000
--- a/packages/pipeline/src/data_sources/etherscan/index.ts
+++ /dev/null
@@ -1,52 +0,0 @@
-import { default as axios } from 'axios';
-import { BlockParam, BlockParamLiteral } from 'ethereum-types';
-
-const ETHERSCAN_URL = 'https://api.etherscan.io/api';
-
-export class Etherscan {
-    private readonly _apiKey: string;
-    constructor(apiKey: string) {
-        this._apiKey = apiKey;
-    }
-
-    /**
-     * Gets the raw events for a specific contract and block range.
-     * @param contractAddress  The address of the contract to get the events for.
-     * @param fromBlock The start of the block range to get events for (inclusive).
-     * @param toBlock The end of the block range to get events for (inclusive).
-     * @returns A list of decoded events.
-     */
-    public async getContractEventsAsync(
-        contractAddress: string,
-        fromBlock: BlockParam = BlockParamLiteral.Earliest,
-        toBlock: BlockParam = BlockParamLiteral.Latest,
-    ): Promise<EventsResponse> {
-        const fullURL = `${ETHERSCAN_URL}?module=logs&action=getLogs&address=${contractAddress}&fromBlock=${fromBlock}&toBlock=${toBlock}&apikey=${
-            this._apiKey
-        }`;
-        const resp = await axios.get<EventsResponse>(fullURL);
-        // TODO(albrow): Check response code.
-        return resp.data;
-    }
-}
-
-// Raw events response from etherescan.io
-export interface EventsResponse {
-    status: string;
-    message: string;
-    result: EventsResponseResult[];
-}
-
-// Events as represented in the response from etherscan.io
-export interface EventsResponseResult {
-    address: string;
-    topics: string[];
-    data: string;
-    blockNumber: string;
-    timeStamp: string;
-    gasPrice: string;
-    gasUsed: string;
-    logIndex: string;
-    transactionHash: string;
-    transactionIndex: string;
-}
diff --git a/packages/pipeline/src/data_types/events/event_utils.ts b/packages/pipeline/src/data_types/events/event_utils.ts
deleted file mode 100644
index 6be964807..000000000
--- a/packages/pipeline/src/data_types/events/event_utils.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import { AbiDecoder } from '@0xproject/utils';
-import { AbiDefinition, LogEntry, LogWithDecodedArgs } from 'ethereum-types';
-
-import { EventsResponseResult } from '../../data_sources/etherscan';
-
-const hexRadix = 16;
-
-function hexToInt(hex: string): number {
-    return parseInt(hex.replace('0x', ''), hexRadix);
-}
-
-// Converts a raw event response to a LogEntry
-export function convertResponseToLogEntry(result: EventsResponseResult): LogEntry {
-    return {
-        logIndex: hexToInt(result.logIndex),
-        transactionIndex: hexToInt(result.transactionIndex),
-        transactionHash: result.transactionHash,
-        blockHash: '',
-        blockNumber: hexToInt(result.blockNumber),
-        address: result.address,
-        data: result.data,
-        topics: result.topics,
-    };
-}
-
-// Decodes a LogEntry into a LogWithDecodedArgs
-export function decodeLogEntry<EventArgsType>(
-    contractAbi: AbiDefinition[],
-    log: LogEntry,
-): LogWithDecodedArgs<EventArgsType> {
-    const abiDecoder = new AbiDecoder([contractAbi]);
-    const logWithDecodedArgs = abiDecoder.tryToDecodeLogOrNoop<EventArgsType>(log);
-    // tslint:disable-next-line:no-unnecessary-type-assertion
-    return logWithDecodedArgs as LogWithDecodedArgs<EventArgsType>;
-}
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
index a1dbb35ff..77c92cc34 100644
--- a/packages/pipeline/src/index.ts
+++ b/packages/pipeline/src/index.ts
@@ -1,52 +1,43 @@
 import { HttpClient } from '@0xproject/connect';
+import { web3Factory } from '@0xproject/dev-utils';
 import 'reflect-metadata';
 import { Connection, createConnection } from 'typeorm';
 
-import { Etherscan } from './data_sources/etherscan';
-import { parseExchangeEvents } from './data_types/events/exchange_events';
-import { parseSraOrders } from './data_types/sra_orders';
-import { ExchangeCancelEvent } from './entities/ExchangeCancelEvent';
-import { ExchangeCancelUpToEvent } from './entities/ExchangeCancelUpToEvent';
-import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
+import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events';
 import { SraOrder } from './entities/SraOrder';
 import { config } from './ormconfig';
-
-const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string);
-const EXCHANGE_ADDRESS = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b';
+import { parseExchangeEvents } from './parsers/events';
+import { parseSraOrders } from './parsers/sra_orders';
 
 let connection: Connection;
 
 (async () => {
     connection = await createConnection(config);
     await getExchangeEventsAsync();
-    await getSraOrdersAsync();
+    // await getSraOrdersAsync();
 })();
 
+// TODO(albrow): Separately: Errors do not appear to be handled correctly. If you use the
+// wrong rpcUrl it just returns early with no error.
 async function getExchangeEventsAsync(): Promise<void> {
-    const fillRepository = connection.getRepository(ExchangeFillEvent);
-    const cancelRepository = connection.getRepository(ExchangeCancelEvent);
-    const cancelUpToRepository = connection.getRepository(ExchangeCancelUpToEvent);
-    console.log(
-        `found ${(await fillRepository.count()) +
-            (await cancelRepository.count()) +
-            (await cancelUpToRepository.count())} existing events`,
-    );
-    const rawEvents = await etherscan.getContractEventsAsync(EXCHANGE_ADDRESS);
-    const events = parseExchangeEvents(rawEvents);
+    const provider = web3Factory.getRpcProvider({
+        rpcUrl: 'https://mainnet.infura.io',
+    });
+    const exchangeEvents = new ExchangeEventsSource(provider, 1);
+    const eventLogs = await exchangeEvents.getFillEventsAsync();
+    const events = parseExchangeEvents(eventLogs);
+    console.log('Got events: ' + events.length);
     for (const event of events) {
         await event.save();
     }
-    console.log(
-        `now there are ${(await fillRepository.count()) +
-            (await cancelRepository.count()) +
-            (await cancelUpToRepository.count())} total events`,
-    );
+    console.log('Saved events.');
+    console.log('Exiting process');
+    process.exit(0);
 }
 
 async function getSraOrdersAsync(): Promise<void> {
     const orderRepository = connection.getRepository(SraOrder);
     console.log(`found ${await orderRepository.count()} existing orders`);
-
     const sraUrl = 'https://api.radarrelay.com/0x/v2';
     const connect = new HttpClient(sraUrl);
     const rawOrders = await connect.getOrdersAsync();
diff --git a/packages/pipeline/src/data_types/events/exchange_events.ts b/packages/pipeline/src/parsers/events/index.ts
index 30ef058f3..66f382dda 100644
--- a/packages/pipeline/src/data_types/events/exchange_events.ts
+++ b/packages/pipeline/src/parsers/events/index.ts
@@ -1,4 +1,3 @@
-import { Exchange } from '@0xproject/contract-artifacts';
 import {
     ExchangeCancelEventArgs,
     ExchangeCancelUpToEventArgs,
@@ -10,34 +9,16 @@ import { AssetProxyId, ERC721AssetData } from '@0xproject/types';
 import { LogWithDecodedArgs } from 'ethereum-types';
 import * as R from 'ramda';
 
-import { EventsResponse } from '../../data_sources/etherscan';
 import { ExchangeCancelEvent } from '../../entities/ExchangeCancelEvent';
 import { ExchangeCancelUpToEvent } from '../../entities/ExchangeCancelUpToEvent';
 import { ExchangeFillEvent } from '../../entities/ExchangeFillEvent';
 import { bigNumbertoStringOrNull } from '../../utils';
 
-import { convertResponseToLogEntry, decodeLogEntry } from './event_utils';
-
 export type ExchangeEventEntity = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent;
 
-export function parseExchangeEvents(rawEventsResponse: EventsResponse): ExchangeEventEntity[] {
-    const logEntries = R.map(convertResponseToLogEntry, rawEventsResponse.result);
-    const decodedLogEntries = R.map(
-        eventResponse => decodeLogEntry<ExchangeEventArgs>(Exchange.compilerOutput.abi, eventResponse),
-        logEntries,
-    );
-    const filteredLogEntries = R.filter(shouldIncludeLogEntry, decodedLogEntries);
-    return R.map(_convertToEntity, filteredLogEntries);
-}
-
-export function shouldIncludeLogEntry(logEntry: LogWithDecodedArgs<ExchangeEventArgs>): boolean {
-    if (!R.contains(logEntry.event, ['Fill', 'Cancel', 'CancelUpTo'])) {
-        return false;
-    } else if (logEntry.logIndex == null || isNaN(logEntry.logIndex)) {
-        return false;
-    }
-    return true;
-}
+export const parseExchangeEvents: (
+    eventLogs: Array<LogWithDecodedArgs<ExchangeEventArgs>>,
+) => ExchangeEventEntity[] = R.map(_convertToEntity);
 
 export function _convertToEntity(eventLog: LogWithDecodedArgs<ExchangeEventArgs>): ExchangeEventEntity {
     switch (eventLog.event) {
diff --git a/packages/pipeline/src/data_types/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts
index fb2b74dfe..fb2b74dfe 100644
--- a/packages/pipeline/src/data_types/sra_orders/index.ts
+++ b/packages/pipeline/src/parsers/sra_orders/index.ts