Run publish/installation tests in CircleCI (#951)

feat(monorepo-scripts): Run publish tests in CircleCI

4 files changed, 134 insertions, 46 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 43e542a86..a6fb511b9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -56,6 +56,16 @@ jobs:
       # HACK(albrow): we need to sleep 10 seconds to ensure the devnet is
       # initialized
       - run: sleep 10 && TEST_PROVIDER=geth yarn wsrun test contracts
+  test-publish:
+    docker:
+      - image: circleci/node:9
+      - image: verdaccio/verdaccio
+    working_directory: ~/repo
+    steps:
+      - restore_cache:
+          keys:
+            - repo-{{ .Environment.CIRCLE_SHA1 }}
+      - run: yarn test:publish:circleci
   test-rest:
     docker:
       - image: circleci/node:9
@@ -232,6 +242,9 @@ workflows:
       - static-tests:
           requires:
             - build
+      - test-publish:
+          requires:
+            - build
       - submit-coverage:
           requires:
             - test-rest
diff --git a/package.json b/package.json
index 5183e2c04..9b8146be8 100644
--- a/package.json
+++ b/package.json
@@ -14,6 +14,8 @@
         "report_coverage": "lcov-result-merger 'packages/*/coverage/lcov.info' | coveralls",
         "test:installation": "node ./packages/monorepo-scripts/lib/test_installation.js",
         "test:installation:local": "IS_LOCAL_PUBLISH=true node ./packages/monorepo-scripts/lib/test_installation.js",
+        "test:publish:circleci:comment": "HACK(albrow) We need an automated way to login to npm and echo+sleep piped to stdin was the only way I could find to do it.",
+        "test:publish:circleci": "{ echo \"test\"; sleep 1; echo \"test\"; sleep 1; echo \"test@example.com\"; } | npm login --registry=http://localhost:4873 && IS_LOCAL_PUBLISH=true run-s script:publish test:installation:local",
         "run:publish": "run-s install:all build:monorepo_scripts script:prepublish_checks rebuild:no_website script:publish",
         "run:publish:local": "IS_LOCAL_PUBLISH=true yarn run:publish",
         "script:prepublish_checks": "node ./packages/monorepo-scripts/lib/prepublish_checks.js",
diff --git a/packages/monorepo-scripts/package.json b/packages/monorepo-scripts/package.json
index b5fae9835..7d83d5a51 100644
--- a/packages/monorepo-scripts/package.json
+++ b/packages/monorepo-scripts/package.json
@@ -30,6 +30,7 @@
     "homepage": "https://github.com/0xProject/0x-monorepo/packages/monorepo-scripts/README.md",
     "devDependencies": {
         "@types/glob": "^5.0.33",
+        "@types/mkdirp": "^0.5.2",
         "@types/node": "^8.0.53",
         "@types/opn": "^5.1.0",
         "@types/rimraf": "^2.0.2",
@@ -50,6 +51,7 @@
         "glob": "^7.1.2",
         "isomorphic-fetch": "2.2.1",
         "lodash": "^4.17.5",
+        "mkdirp": "^0.5.1",
         "moment": "2.21.0",
         "opn": "^5.3.0",
         "promisify-child-process": "^1.0.5",
diff --git a/packages/monorepo-scripts/src/test_installation.ts b/packages/monorepo-scripts/src/test_installation.ts
index a8ddf0c58..87c4ad1d7 100644
--- a/packages/monorepo-scripts/src/test_installation.ts
+++ b/packages/monorepo-scripts/src/test_installation.ts
@@ -2,10 +2,13 @@
 
 import * as fs from 'fs';
 import * as _ from 'lodash';
+import * as mkdirp from 'mkdirp';
 import * as path from 'path';
 import { exec as execAsync } from 'promisify-child-process';
 import * as rimraf from 'rimraf';
+import { promisify } from 'util';
 
+import { Package } from './types';
 import { utils } from './utils/utils';
 
 // Packages might not be runnable if they are command-line tools or only run in browsers.
@@ -16,64 +19,132 @@ const UNRUNNABLE_PACKAGES = [
     '@0xproject/react-docs',
 ];
 
+const mkdirpAsync = promisify(mkdirp);
+const rimrafAsync = promisify(rimraf);
+const writeFileAsync = promisify(fs.writeFile);
+
+interface PackageErr {
+    packageName: string;
+    error: ExecError;
+}
+
+interface ExecError {
+    message: string;
+    stack: string;
+    stderr: string;
+    stdout: string;
+}
+
+// returns the index for the given package name.
+function findPackageIndex(packages: Package[], packageName: string): number {
+    return _.findIndex(packages, pkg => pkg.packageJson.name === packageName);
+}
+
+function logIfDefined(x: any): void {
+    if (!_.isUndefined(x)) {
+        utils.log(x);
+    }
+}
+
 (async () => {
     const IS_LOCAL_PUBLISH = process.env.IS_LOCAL_PUBLISH === 'true';
     const registry = IS_LOCAL_PUBLISH ? 'http://localhost:4873/' : 'https://registry.npmjs.org/';
     const monorepoRootPath = path.join(__dirname, '../../..');
-    const packages = utils.getTopologicallySortedPackages(monorepoRootPath);
+    const packages = utils.getPackages(monorepoRootPath);
     const installablePackages = _.filter(
         packages,
         pkg => !pkg.packageJson.private && !_.isUndefined(pkg.packageJson.main) && pkg.packageJson.main.endsWith('.js'),
     );
     utils.log('Testing packages:');
     _.map(installablePackages, pkg => utils.log(`* ${pkg.packageJson.name}`));
+    // Run all package tests asynchronously and push promises into an array so
+    // we can wait for all of them to resolve.
+    const promises: Array<Promise<void>> = [];
+    const errors: PackageErr[] = [];
     for (const installablePackage of installablePackages) {
-        const changelogPath = path.join(installablePackage.location, 'CHANGELOG.json');
-        const lastChangelogVersion = JSON.parse(fs.readFileSync(changelogPath).toString())[0].version;
-        const packageName = installablePackage.packageJson.name;
-        utils.log(`Testing ${packageName}@${lastChangelogVersion}`);
-        const testDirectory = path.join(monorepoRootPath, '../test-env');
-        rimraf.sync(testDirectory);
-        fs.mkdirSync(testDirectory);
-        await execAsync('yarn init --yes', { cwd: testDirectory });
-        const npmrcFilePath = path.join(testDirectory, '.npmrc');
-        fs.writeFileSync(npmrcFilePath, `registry=${registry}`);
-        utils.log(`Installing ${packageName}@${lastChangelogVersion}`);
-        await execAsync(`npm install --save ${packageName}@${lastChangelogVersion} --registry=${registry}`, {
-            cwd: testDirectory,
+        const packagePromise = testInstallPackageAsync(monorepoRootPath, registry, installablePackage).catch(error => {
+            errors.push({ packageName: installablePackage.packageJson.name, error });
         });
-        const indexFilePath = path.join(testDirectory, 'index.ts');
-        fs.writeFileSync(indexFilePath, `import * as Package from '${packageName}';\nconsole.log(Package);\n`);
-        const tsConfig = {
-            compilerOptions: {
-                typeRoots: ['node_modules/@0xproject/typescript-typings/types', 'node_modules/@types'],
-                module: 'commonjs',
-                target: 'es5',
-                lib: ['es2017', 'dom'],
-                declaration: true,
-                noImplicitReturns: true,
-                pretty: true,
-                strict: true,
-            },
-            include: ['index.ts'],
-        };
-        const tsconfigFilePath = path.join(testDirectory, 'tsconfig.json');
-        fs.writeFileSync(tsconfigFilePath, JSON.stringify(tsConfig, null, '\t'));
-        utils.log(`Compiling ${packageName}`);
-        const tscBinaryPath = path.join(monorepoRootPath, './node_modules/typescript/bin/tsc');
-        await execAsync(tscBinaryPath, { cwd: testDirectory });
-        utils.log(`Successfully compiled with ${packageName} as a dependency`);
-        const isUnrunnablePkg = _.includes(UNRUNNABLE_PACKAGES, packageName);
-        if (!isUnrunnablePkg) {
-            const transpiledIndexFilePath = path.join(testDirectory, 'index.js');
-            utils.log(`Running test script with ${packageName} imported`);
-            await execAsync(`node ${transpiledIndexFilePath}`);
-            utils.log(`Successfilly ran test script with ${packageName} imported`);
-        }
-        rimraf.sync(testDirectory);
+        promises.push(packagePromise);
+    }
+    await Promise.all(promises);
+    if (errors.length > 0) {
+        // We sort error messages according to package topology so that we can
+        // them in a more intuitive order. E.g. if package A has an error and
+        // package B imports it, the tests for both package A and package B will
+        // fail. But package B only fails because of an error in package A.
+        // Since the error in package A is the root cause, we log it first.
+        const topologicallySortedPackages = utils.getTopologicallySortedPackages(monorepoRootPath);
+        const topologicallySortedErrors = _.sortBy(errors, packageErr =>
+            findPackageIndex(topologicallySortedPackages, packageErr.packageName),
+        );
+        _.forEach(topologicallySortedErrors, packageError => {
+            utils.log(`ERROR in package ${packageError.packageName}:`);
+            logIfDefined(packageError.error.message);
+            logIfDefined(packageError.error.stderr);
+            logIfDefined(packageError.error.stdout);
+            logIfDefined(packageError.error.stack);
+        });
+        process.exit(0);
     }
 })().catch(err => {
-    utils.log(err.stderr);
-    utils.log(err.stdout);
-    process.exit(1);
+    utils.log(`Unexpected error: ${err.message}`);
+    process.exit(0);
 });
+
+async function testInstallPackageAsync(
+    monorepoRootPath: string,
+    registry: string,
+    installablePackage: Package,
+): Promise<void> {
+    const changelogPath = path.join(installablePackage.location, 'CHANGELOG.json');
+    const lastChangelogVersion = JSON.parse(fs.readFileSync(changelogPath).toString())[0].version;
+    const packageName = installablePackage.packageJson.name;
+    utils.log(`Testing ${packageName}@${lastChangelogVersion}`);
+    const packageDirName = path.join(...(packageName + '-test').split('/'));
+    const testDirectory = path.join(
+        monorepoRootPath,
+        'packages',
+        'monorepo-scripts',
+        '.installation-test',
+        packageDirName,
+    );
+    await rimrafAsync(testDirectory);
+    await mkdirpAsync(testDirectory);
+    await execAsync('yarn init --yes', { cwd: testDirectory });
+    const npmrcFilePath = path.join(testDirectory, '.npmrc');
+    await writeFileAsync(npmrcFilePath, `registry=${registry}`);
+    utils.log(`Installing ${packageName}@${lastChangelogVersion}`);
+    await execAsync(`npm install --save ${packageName}@${lastChangelogVersion} --registry=${registry}`, {
+        cwd: testDirectory,
+    });
+    const indexFilePath = path.join(testDirectory, 'index.ts');
+    await writeFileAsync(indexFilePath, `import * as Package from '${packageName}';\nconsole.log(Package);\n`);
+    const tsConfig = {
+        compilerOptions: {
+            typeRoots: ['node_modules/@0xproject/typescript-typings/types', 'node_modules/@types'],
+            module: 'commonjs',
+            target: 'es5',
+            lib: ['es2017', 'dom'],
+            declaration: true,
+            noImplicitReturns: true,
+            pretty: true,
+            strict: true,
+        },
+        include: ['index.ts'],
+    };
+    const tsconfigFilePath = path.join(testDirectory, 'tsconfig.json');
+    await writeFileAsync(tsconfigFilePath, JSON.stringify(tsConfig, null, '\t'));
+    utils.log(`Compiling ${packageName}`);
+    const tscBinaryPath = path.join(monorepoRootPath, './node_modules/typescript/bin/tsc');
+    await execAsync(tscBinaryPath, { cwd: testDirectory });
+    utils.log(`Successfully compiled with ${packageName} as a dependency`);
+    const isUnrunnablePkg = _.includes(UNRUNNABLE_PACKAGES, packageName);
+    if (!isUnrunnablePkg) {
+        const transpiledIndexFilePath = path.join(testDirectory, 'index.js');
+        utils.log(`Running test script with ${packageName} imported`);
+        await execAsync(`node ${transpiledIndexFilePath}`);
+        utils.log(`Successfilly ran test script with ${packageName} imported`);
+    }
+    await rimrafAsync(testDirectory);
+}