From 629a8d632801c4861824b7d0423792d167b8564d Mon Sep 17 00:00:00 2001 From: "F. Eugene Aumson" Date: Fri, 8 Feb 2019 13:21:41 -0500 Subject: Add migration to fix exchange events primary keys (#1593) * Add migration to fix exchange events primary key * correct comment: "foreign key" -> "primary key" * Refine hack to handle only the expected error * Add tx hash to erc20 approval events primary key --- ...479172800-AddTxHashToExchangeEventPrimaryKey.ts | 35 ++++++++++++++++++++++ ...6238-AddTxHashToERC20ApprovalEventPrimaryKey.ts | 31 +++++++++++++++++++ .../pipeline/src/entities/erc20_approval_event.ts | 2 +- .../pipeline/src/entities/exchange_cancel_event.ts | 2 +- .../src/entities/exchange_cancel_up_to_event.ts | 2 +- .../pipeline/src/entities/exchange_fill_event.ts | 2 +- .../pipeline/src/scripts/pull_exchange_events.ts | 12 ++++++-- 7 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 packages/pipeline/migrations/1549479172800-AddTxHashToExchangeEventPrimaryKey.ts create mode 100644 packages/pipeline/migrations/1549499426238-AddTxHashToERC20ApprovalEventPrimaryKey.ts (limited to 'packages/pipeline') diff --git a/packages/pipeline/migrations/1549479172800-AddTxHashToExchangeEventPrimaryKey.ts b/packages/pipeline/migrations/1549479172800-AddTxHashToExchangeEventPrimaryKey.ts new file mode 100644 index 000000000..d6ea6c47b --- /dev/null +++ b/packages/pipeline/migrations/1549479172800-AddTxHashToExchangeEventPrimaryKey.ts @@ -0,0 +1,35 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +const tableNames = ['exchange_cancel_events', 'exchange_cancel_up_to_events', 'exchange_fill_events']; + +const oldPrimaryColumns = ['contract_address', 'log_index', 'block_number']; + +const newPrimaryColumns = ['transaction_hash']; + +async function updatePrimaryKeysAsync(queryRunner: QueryRunner, columnNames: string[]): Promise { + for (const tableName of tableNames) { + const table = await queryRunner.getTable(`raw.${tableName}`); + if (table === undefined) { + throw new Error(`Couldn't get table 'raw.${tableName}'`); + } + const columns = []; + for (const columnName of columnNames) { + const column = table.findColumnByName(columnName); + if (column === undefined) { + throw new Error(`Couldn't get column '${columnName}' from table 'raw.${tableName}'`); + } + columns.push(column); + } + await queryRunner.updatePrimaryKeys(table, columns); + } +} + +export class AddTxHashToExchangeEventPrimaryKey1549479172800 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await updatePrimaryKeysAsync(queryRunner, oldPrimaryColumns.concat(newPrimaryColumns)); + } + + public async down(queryRunner: QueryRunner): Promise { + await updatePrimaryKeysAsync(queryRunner, oldPrimaryColumns); + } +} diff --git a/packages/pipeline/migrations/1549499426238-AddTxHashToERC20ApprovalEventPrimaryKey.ts b/packages/pipeline/migrations/1549499426238-AddTxHashToERC20ApprovalEventPrimaryKey.ts new file mode 100644 index 000000000..874713e67 --- /dev/null +++ b/packages/pipeline/migrations/1549499426238-AddTxHashToERC20ApprovalEventPrimaryKey.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +const oldPrimaryColumns = ['token_address', 'log_index', 'block_number']; + +const newPrimaryColumns = ['transaction_hash']; + +async function updatePrimaryKeysAsync(queryRunner: QueryRunner, columnNames: string[]): Promise { + const table = await queryRunner.getTable(`raw.erc20_approval_events`); + if (table === undefined) { + throw new Error(`Couldn't get table 'raw.erc20_approval_events'`); + } + const columns = []; + for (const columnName of columnNames) { + const column = table.findColumnByName(columnName); + if (column === undefined) { + throw new Error(`Couldn't get column '${columnName}' from table 'raw.erc20_approval_events'`); + } + columns.push(column); + } + await queryRunner.updatePrimaryKeys(table, columns); +} + +export class AddTxHashToERC20ApprovalEventPrimaryKey1549499426238 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await updatePrimaryKeysAsync(queryRunner, oldPrimaryColumns.concat(newPrimaryColumns)); + } + + public async down(queryRunner: QueryRunner): Promise { + await updatePrimaryKeysAsync(queryRunner, oldPrimaryColumns); + } +} diff --git a/packages/pipeline/src/entities/erc20_approval_event.ts b/packages/pipeline/src/entities/erc20_approval_event.ts index 69cdfcb0b..ee5e621d2 100644 --- a/packages/pipeline/src/entities/erc20_approval_event.ts +++ b/packages/pipeline/src/entities/erc20_approval_event.ts @@ -15,7 +15,7 @@ export class ERC20ApprovalEvent { @Column({ name: 'raw_data' }) public rawData!: string; - @Column({ name: 'transaction_hash' }) + @PrimaryColumn({ name: 'transaction_hash' }) public transactionHash!: string; @Column({ name: 'owner_address' }) public ownerAddress!: string; diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts index 38f99c903..a86194920 100644 --- a/packages/pipeline/src/entities/exchange_cancel_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_event.ts @@ -15,7 +15,7 @@ export class ExchangeCancelEvent { @Column({ name: 'raw_data' }) public rawData!: string; - @Column({ name: 'transaction_hash' }) + @PrimaryColumn({ name: 'transaction_hash' }) public transactionHash!: string; @Column({ name: 'maker_address' }) public makerAddress!: string; diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts index 27580305e..f24aea23a 100644 --- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts @@ -15,7 +15,7 @@ export class ExchangeCancelUpToEvent { @Column({ name: 'raw_data' }) public rawData!: string; - @Column({ name: 'transaction_hash' }) + @PrimaryColumn({ name: 'transaction_hash' }) public transactionHash!: string; @Column({ name: 'maker_address' }) public makerAddress!: string; diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts index 9b7727615..52111711e 100644 --- a/packages/pipeline/src/entities/exchange_fill_event.ts +++ b/packages/pipeline/src/entities/exchange_fill_event.ts @@ -16,7 +16,7 @@ export class ExchangeFillEvent { @Column({ name: 'raw_data' }) public rawData!: string; - @Column({ name: 'transaction_hash' }) + @PrimaryColumn({ name: 'transaction_hash' }) public transactionHash!: string; @Column({ name: 'maker_address' }) public makerAddress!: string; diff --git a/packages/pipeline/src/scripts/pull_exchange_events.ts b/packages/pipeline/src/scripts/pull_exchange_events.ts index f8ce4038d..c2c56da6b 100644 --- a/packages/pipeline/src/scripts/pull_exchange_events.ts +++ b/packages/pipeline/src/scripts/pull_exchange_events.ts @@ -112,15 +112,20 @@ async function saveIndividuallyWithFallbackAsync( events: T[], ): Promise { // Note(albrow): This is a temporary hack because `save` is not working as - // documented and is causing a foreign key constraint violation. Hopefully + // documented and is causing a primary key constraint violation. Hopefully // can remove later because this "poor man's upsert" implementation operates // on one event at a time and is therefore much slower. for (const event of events) { try { // First try an insert. await repository.insert(event); - } catch { - // If it fails, assume it was a foreign key constraint error and try + } catch (err) { + if (err.message.includes('duplicate key value violates unique constraint')) { + logUtils.log("Ignore the preceeding INSERT failure; it's not unexpected"); + } else { + throw err; + } + // If it fails, assume it was a primary key constraint error and try // doing an update instead. // Note(albrow): Unfortunately the `as any` hack here seems // required. I can't figure out how to convince the type-checker @@ -132,6 +137,7 @@ async function saveIndividuallyWithFallbackAsync( contractAddress: event.contractAddress, blockNumber: event.blockNumber, logIndex: event.logIndex, + transactionHash: event.transactionHash, } as any, event as any, ); -- cgit