pull OHLCV records from Crypto Compare (#1349)

* [WIP] pull OHLCV records from Crypto Compare

* lint

* refactor to pull logic out of script and into modules

* add entity test for ohlcv_external entity

* implement rate limit and chronological backfill for ohlcv

* add unit tests; cleanup variable names

* Fetch OHLCV pairs params from events table

* better method names

* fix outdated test

* lint

* Clean up after review

* oops

* fix failing test

* better filtering of most recent records

* fix bug when generating pairs

* fix default earliest backfill date

* fix bug with retrieving backfill time

* prettier

3 files changed, 144 insertions, 0 deletions

diff --git a/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts
new file mode 100644
index 000000000..cb374bbb1
--- /dev/null
+++ b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts
@@ -0,0 +1,47 @@
+import * as chai from 'chai';
+import 'mocha';
+import * as R from 'ramda';
+
+import { CryptoCompareOHLCVSource } from '../../../src/data_sources/ohlcv_external/crypto_compare';
+import { TradingPair } from '../../../src/utils/get_ohlcv_trading_pairs';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('ohlcv_external data source (Crypto Compare)', () => {
+    describe('generateBackfillIntervals', () => {
+        it('generates pairs with intervals to query', () => {
+            const source = new CryptoCompareOHLCVSource();
+            const pair: TradingPair = {
+                fromSymbol: 'ETH',
+                toSymbol: 'ZRX',
+                latestSavedTime: new Date().getTime() - source.interval * 2,
+            };
+
+            const expected = [
+                pair,
+                R.merge(pair, { latestSavedTime: pair.latestSavedTime + source.interval }),
+                R.merge(pair, { latestSavedTime: pair.latestSavedTime + source.interval * 2 }),
+            ];
+
+            const actual = source.generateBackfillIntervals(pair);
+            expect(actual).deep.equal(expected);
+        });
+
+        it('returns single pair if no backfill is needed', () => {
+            const source = new CryptoCompareOHLCVSource();
+            const pair: TradingPair = {
+                fromSymbol: 'ETH',
+                toSymbol: 'ZRX',
+                latestSavedTime: new Date().getTime() - source.interval + 5000,
+            };
+
+            const expected = [pair];
+
+            const actual = source.generateBackfillIntervals(pair);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/entities/ohlcv_external_test.ts b/packages/pipeline/test/entities/ohlcv_external_test.ts
new file mode 100644
index 000000000..8b995db50
--- /dev/null
+++ b/packages/pipeline/test/entities/ohlcv_external_test.ts
@@ -0,0 +1,35 @@
+import 'mocha';
+import 'reflect-metadata';
+
+import { OHLCVExternal } from '../../src/entities';
+import { createDbConnectionOnceAsync } from '../db_setup';
+import { chaiSetup } from '../utils/chai_setup';
+
+import { testSaveAndFindEntityAsync } from './util';
+
+chaiSetup.configure();
+
+const ohlcvExternal: OHLCVExternal = {
+    exchange: 'CCCAGG',
+    fromSymbol: 'ETH',
+    toSymbol: 'ZRX',
+    startTime: 1543352400000,
+    endTime: 1543356000000,
+    open: 307.41,
+    close: 310.08,
+    low: 304.6,
+    high: 310.27,
+    volumeFrom: 904.6,
+    volumeTo: 278238.5,
+    source: 'Crypto Compare',
+    observedTimestamp: 1543442338074,
+};
+
+// tslint:disable:custom-no-magic-numbers
+describe('OHLCVExternal entity', () => {
+    it('save/find', async () => {
+        const connection = await createDbConnectionOnceAsync();
+        const repository = connection.getRepository(OHLCVExternal);
+        await testSaveAndFindEntityAsync(repository, ohlcvExternal);
+    });
+});
diff --git a/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts b/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts
new file mode 100644
index 000000000..118cafc5e
--- /dev/null
+++ b/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts
@@ -0,0 +1,62 @@
+import * as chai from 'chai';
+import 'mocha';
+import * as R from 'ramda';
+
+import { CryptoCompareOHLCVRecord } from '../../../src/data_sources/ohlcv_external/crypto_compare';
+import { OHLCVExternal } from '../../../src/entities';
+import { OHLCVMetadata, parseRecords } from '../../../src/parsers/ohlcv_external/crypto_compare';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('ohlcv_external parser (Crypto Compare)', () => {
+    describe('parseRecords', () => {
+        const record: CryptoCompareOHLCVRecord = {
+            time: 200,
+            close: 100,
+            high: 101,
+            low: 99,
+            open: 98,
+            volumefrom: 1234,
+            volumeto: 4321,
+        };
+
+        const metadata: OHLCVMetadata = {
+            fromSymbol: 'ETH',
+            toSymbol: 'ZRX',
+            exchange: 'CCCAGG',
+            source: 'CryptoCompare',
+            observedTimestamp: new Date().getTime(),
+            interval: 100000,
+        };
+
+        const entity = new OHLCVExternal();
+        entity.exchange = metadata.exchange;
+        entity.fromSymbol = metadata.fromSymbol;
+        entity.toSymbol = metadata.toSymbol;
+        entity.startTime = 100000;
+        entity.endTime = 200000;
+        entity.open = record.open;
+        entity.close = record.close;
+        entity.low = record.low;
+        entity.high = record.high;
+        entity.volumeFrom = record.volumefrom;
+        entity.volumeTo = record.volumeto;
+        entity.source = metadata.source;
+        entity.observedTimestamp = metadata.observedTimestamp;
+
+        it('converts Crypto Compare OHLCV records to OHLCVExternal entity', () => {
+            const input = [record, R.merge(record, { time: 300 }), R.merge(record, { time: 400 })];
+            const expected = [
+                entity,
+                R.merge(entity, { startTime: 200000, endTime: 300000 }),
+                R.merge(entity, { startTime: 300000, endTime: 400000 }),
+            ];
+
+            const actual = parseRecords(input, metadata);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});