Merge pull request #489 from 0xProject/refactor/publishProcess

Automate NPM Publish Process

11 files changed, 408 insertions, 17 deletions

diff --git a/packages/monorepo-scripts/README.md b/packages/monorepo-scripts/README.md
index ea1d43bd5..dbdfdf135 100644
--- a/packages/monorepo-scripts/README.md
+++ b/packages/monorepo-scripts/README.md
@@ -9,7 +9,7 @@ This repository contains a few helpful scripts for working with this mono repo.
 In order to reduce the size of this repo, we try and use the same versions of dependencies between packages. To make it easier to discover version discrepancies between packages, you can run:
 
 ```bash
-yarn deps_versions
+yarn scripts:deps_versions
 ```
 
 This will list out any dependencies that differ in versions between packages.
diff --git a/packages/monorepo-scripts/package.json b/packages/monorepo-scripts/package.json
index 5333e6cad..ed263d5cb 100644
--- a/packages/monorepo-scripts/package.json
+++ b/packages/monorepo-scripts/package.json
@@ -6,10 +6,14 @@
     "types": "lib/index.d.ts",
     "scripts": {
         "build:watch": "tsc -w",
-        "deps_versions": "node ./lib/deps_versions.js",
         "lint": "tslint --project . 'src/**/*.ts'",
         "clean": "shx rm -rf lib",
-        "build": "tsc"
+        "build": "tsc",
+        "test:publish": "run-s build script:publish",
+        "convert_changelogs": "run-s build script:convert_changelogs",
+        "script:deps_versions": "node ./lib/deps_versions.js",
+        "script:publish": "IS_DRY_RUN=true node ./lib/publish.js",
+        "script:convert_changelogs": "node ./lib/convert_changelogs.js"
     },
     "repository": {
         "type": "git",
@@ -27,6 +31,7 @@
         "@types/node": "^8.0.53",
         "@types/rimraf": "^2.0.2",
         "lerna-get-packages": "^1.0.0",
+        "npm-run-all": "^4.1.2",
         "shx": "^0.2.2",
         "tslint": "5.8.0",
         "typescript": "2.7.1"
@@ -37,10 +42,12 @@
         "es6-promisify": "^5.0.0",
         "glob": "^7.1.2",
         "lodash": "^4.17.4",
+        "moment": "2.21.0",
         "promisify-child-process": "^1.0.5",
         "publish-release": "0xproject/publish-release",
         "rimraf": "^2.6.2",
-        "semver-sort": "^0.0.4"
+        "semver-diff": "^2.1.0",
+        "semver-sort": "0.0.4"
     },
     "publishConfig": {
         "access": "public"
diff --git a/packages/monorepo-scripts/src/constants.ts b/packages/monorepo-scripts/src/constants.ts
new file mode 100644
index 000000000..74387a159
--- /dev/null
+++ b/packages/monorepo-scripts/src/constants.ts
@@ -0,0 +1,5 @@
+import * as path from 'path';
+
+export const constants = {
+    monorepoRootPath: path.join(__dirname, '../../..'),
+};
diff --git a/packages/monorepo-scripts/src/convert_changelogs.ts b/packages/monorepo-scripts/src/convert_changelogs.ts
new file mode 100644
index 000000000..b5be14ed8
--- /dev/null
+++ b/packages/monorepo-scripts/src/convert_changelogs.ts
@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+/**
+ * TEMPORARY SCRIPT
+ * This script exists to migrate the legacy CHANGELOG.md to the canonical CHANGELOG.md
+ * TODO: Remove after migration is successful and committed.
+ */
+
+import * as fs from 'fs';
+import lernaGetPackages = require('lerna-get-packages');
+import * as _ from 'lodash';
+import * as moment from 'moment';
+import * as path from 'path';
+import { exec as execAsync } from 'promisify-child-process';
+
+import { constants } from './constants';
+import { Changelog, Changes, UpdatedPackage } from './types';
+import { utils } from './utils';
+
+const HEADER_PRAGMA = '##';
+
+(async () => {
+    const allLernaPackages = lernaGetPackages(constants.monorepoRootPath);
+    const publicLernaPackages = _.filter(allLernaPackages, pkg => !pkg.package.private);
+    for (const lernaPackage of publicLernaPackages) {
+        const changelogMdIfExists = getChangelogMdIfExists(lernaPackage.package.name, lernaPackage.location);
+        if (_.isUndefined(changelogMdIfExists)) {
+            throw new Error(`${lernaPackage.package.name} should have CHANGELOG.md b/c it's public. Add one.`);
+        }
+
+        const lines = changelogMdIfExists.split('\n');
+        const changelogs: Changelog[] = [];
+        let changelog: Changelog = {
+            version: '',
+            changes: [],
+        };
+        /**
+         * Example MD entry:
+         * ## v0.3.1 - _March 18, 2018_
+         *
+         *    * Add TS types for `yargs` (#400)
+         */
+        for (const line of lines) {
+            if (_.startsWith(line, `${HEADER_PRAGMA} `)) {
+                let version = line.substr(4).split(' - ')[0];
+                if (version === '0.x.x') {
+                    version = utils.getNextPatchVersion(lernaPackage.package.version);
+                }
+                const dateStr = line.split('_')[1];
+                let date;
+                if (!_.includes(dateStr, 'TBD')) {
+                    date = moment(dateStr, 'MMMM D, YYYY');
+                }
+                changelog = {
+                    version,
+                    changes: [],
+                };
+                if (!_.isUndefined(date)) {
+                    changelog.timestamp = date.unix();
+                }
+                if (!_.includes(dateStr, 'TBD')) {
+                    changelog.isPublished = true;
+                }
+                changelogs.push(changelog);
+            } else if (_.includes(line, '* ')) {
+                const note = line.split('* ')[1].split(' (#')[0];
+                const prChunk = line.split(' (#')[1];
+                let pr;
+                if (!_.isUndefined(prChunk)) {
+                    pr = prChunk.split(')')[0];
+                }
+                const changes: Changes = {
+                    note,
+                };
+                if (!_.isUndefined(pr)) {
+                    changes.pr = _.parseInt(pr);
+                }
+                changelog.changes.push(changes);
+            }
+        }
+        const changelogJSON = JSON.stringify(changelogs, null, 4);
+        const changelogJSONPath = `${lernaPackage.location}/CHANGELOG.json`;
+        fs.writeFileSync(changelogJSONPath, changelogJSON);
+        await utils.prettifyAsync(changelogJSONPath, constants.monorepoRootPath);
+    }
+})().catch(err => {
+    utils.log(err.stdout);
+    process.exit(1);
+});
+
+function getChangelogMdIfExists(packageName: string, location: string): string | undefined {
+    const changelogPath = path.join(location, 'CHANGELOG.md');
+    let changelogMd: string;
+    try {
+        changelogMd = fs.readFileSync(changelogPath, 'utf-8');
+        return changelogMd;
+    } catch (err) {
+        return undefined;
+    }
+}
diff --git a/packages/monorepo-scripts/src/globals.d.ts b/packages/monorepo-scripts/src/globals.d.ts
index 1d49559f2..c5898d0f5 100644
--- a/packages/monorepo-scripts/src/globals.d.ts
+++ b/packages/monorepo-scripts/src/globals.d.ts
@@ -1,6 +1,7 @@
 declare module 'async-child-process';
 declare module 'publish-release';
 declare module 'es6-promisify';
+declare module 'semver-diff';
 
 // semver-sort declarations
 declare module 'semver-sort' {
@@ -11,6 +12,7 @@ declare interface LernaPackage {
     location: string;
     package: {
         private?: boolean;
+        version: string;
         name: string;
         main?: string;
         config?: {
diff --git a/packages/monorepo-scripts/src/postpublish_utils.ts b/packages/monorepo-scripts/src/postpublish_utils.ts
index 898b00c47..fb1680afd 100644
--- a/packages/monorepo-scripts/src/postpublish_utils.ts
+++ b/packages/monorepo-scripts/src/postpublish_utils.ts
@@ -1,9 +1,12 @@
 import { execAsync } from 'async-child-process';
 import * as promisify from 'es6-promisify';
+import * as fs from 'fs';
 import * as _ from 'lodash';
+import * as path from 'path';
 import * as publishRelease from 'publish-release';
 import semverSort = require('semver-sort');
 
+import { constants } from './constants';
 import { utils } from './utils';
 
 const publishReleaseAsync = promisify(publishRelease);
@@ -88,23 +91,52 @@ export const postpublishUtils = {
         );
     },
     async publishReleaseNotesAsync(cwd: string, packageName: string, version: string, assets: string[]): Promise<void> {
+        const notes = this.getReleaseNotes(packageName);
         const releaseName = this.getReleaseName(packageName, version);
         const tag = this.getTag(packageName, version);
-        utils.log('POSTPUBLISH: Releasing ', releaseName, '...');
         const finalAssets = this.adjustAssetPaths(cwd, assets);
+        utils.log('POSTPUBLISH: Releasing ', releaseName, '...');
         const result = await publishReleaseAsync({
             token: githubPersonalAccessToken,
             owner: '0xProject',
             repo: '0x-monorepo',
             tag,
             name: releaseName,
-            notes: 'N/A',
+            notes,
             draft: false,
             prerelease: false,
             reuseRelease: true,
             reuseDraftOnly: false,
             assets,
         });
+        this.updateChangelogIsPublished(packageName);
+    },
+    getReleaseNotes(packageName: string) {
+        const changelogJSONPath = path.join(constants.monorepoRootPath, 'packages', packageName, 'CHANGELOG.json');
+        const changelogJSON = fs.readFileSync(changelogJSONPath, 'utf-8');
+        const changelogs = JSON.parse(changelogJSON);
+        const latestLog = changelogs[0];
+        if (_.isUndefined(latestLog.isPublished)) {
+            let notes = '';
+            _.each(latestLog.changes, change => {
+                notes = `* ${change.note}`;
+                if (change.pr) {
+                    notes += ` (${change.pr})`;
+                }
+                notes += `\n`;
+            });
+            return notes;
+        }
+        return 'N/A';
+    },
+    updateChangelogIsPublished(packageName: string) {
+        const changelogJSONPath = path.join(constants.monorepoRootPath, 'packages', packageName, 'CHANGELOG.json');
+        const changelogJSON = fs.readFileSync(changelogJSONPath, 'utf-8');
+        const changelogs = JSON.parse(changelogJSON);
+        const latestLog = changelogs[0];
+        latestLog.isPublished = true;
+        changelogs[0] = latestLog;
+        fs.writeFileSync(changelogJSONPath, JSON.stringify(changelogs, null, '\t'));
     },
     getTag(packageName: string, version: string) {
         return `${packageName}@${version}`;
@@ -122,14 +154,16 @@ export const postpublishUtils = {
     },
     adjustFileIncludePaths(fileIncludes: string[], cwd: string): string[] {
         const fileIncludesAdjusted = _.map(fileIncludes, fileInclude => {
-            let path = _.startsWith(fileInclude, './') ? `${cwd}/${fileInclude.substr(2)}` : `${cwd}/${fileInclude}`;
+            let includePath = _.startsWith(fileInclude, './')
+                ? `${cwd}/${fileInclude.substr(2)}`
+                : `${cwd}/${fileInclude}`;
 
             // HACK: tsconfig.json needs wildcard directory endings as `/**/*`
             // but TypeDoc needs it as `/**` in order to pick up files at the root
-            if (_.endsWith(path, '/**/*')) {
-                path = path.slice(0, -2);
+            if (_.endsWith(includePath, '/**/*')) {
+                includePath = includePath.slice(0, -2);
             }
-            return path;
+            return includePath;
         });
         return fileIncludesAdjusted;
     },
diff --git a/packages/monorepo-scripts/src/publish.ts b/packages/monorepo-scripts/src/publish.ts
new file mode 100644
index 000000000..240158c77
--- /dev/null
+++ b/packages/monorepo-scripts/src/publish.ts
@@ -0,0 +1,211 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs';
+import lernaGetPackages = require('lerna-get-packages');
+import * as _ from 'lodash';
+import * as moment from 'moment';
+import * as path from 'path';
+import { exec as execAsync, spawn } from 'promisify-child-process';
+import semverDiff = require('semver-diff');
+import semverSort = require('semver-sort');
+
+import { constants } from './constants';
+import { Changelog, Changes, SemVerIndex, UpdatedPackage } from './types';
+import { utils } from './utils';
+
+const IS_DRY_RUN = process.env.IS_DRY_RUN === 'true';
+const TODAYS_TIMESTAMP = moment().unix();
+const LERNA_EXECUTABLE = './node_modules/lerna/bin/lerna.js';
+const semverNameToIndex: { [semver: string]: number } = {
+    patch: SemVerIndex.Patch,
+    minor: SemVerIndex.Minor,
+    major: SemVerIndex.Major,
+};
+
+(async () => {
+    const updatedPublicPackages = await getPublicLernaUpdatedPackagesAsync();
+    const updatedPackageNames = _.map(updatedPublicPackages, pkg => pkg.name);
+
+    const allLernaPackages = lernaGetPackages(constants.monorepoRootPath);
+    const updatedPublicLernaPackages = _.filter(allLernaPackages, pkg => {
+        return _.includes(updatedPackageNames, pkg.package.name);
+    });
+    const updatedPublicLernaPackageNames = _.map(updatedPublicLernaPackages, pkg => pkg.package.name);
+    utils.log(`Will update CHANGELOGs and publish: \n${updatedPublicLernaPackageNames.join('\n')}\n`);
+
+    const packageToVersionChange: { [name: string]: string } = {};
+    for (const lernaPackage of updatedPublicLernaPackages) {
+        const packageName = lernaPackage.package.name;
+        const changelogJSONPath = path.join(lernaPackage.location, 'CHANGELOG.json');
+        const changelogJSON = getChangelogJSONOrCreateIfMissing(lernaPackage.package.name, changelogJSONPath);
+        let changelogs: Changelog[];
+        try {
+            changelogs = JSON.parse(changelogJSON);
+        } catch (err) {
+            throw new Error(
+                `${lernaPackage.package.name}'s CHANGELOG.json contains invalid JSON. Please fix and try again.`,
+            );
+        }
+
+        const currentVersion = lernaPackage.package.version;
+        const shouldAddNewEntry = shouldAddNewChangelogEntry(changelogs);
+        if (shouldAddNewEntry) {
+            // Create a new entry for a patch version with generic changelog entry.
+            const nextPatchVersion = utils.getNextPatchVersion(currentVersion);
+            const newChangelogEntry: Changelog = {
+                timestamp: TODAYS_TIMESTAMP,
+                version: nextPatchVersion,
+                changes: [
+                    {
+                        note: 'Dependencies updated',
+                    },
+                ],
+            };
+            changelogs = [newChangelogEntry, ...changelogs];
+            packageToVersionChange[packageName] = semverDiff(currentVersion, nextPatchVersion);
+        } else {
+            // Update existing entry with timestamp
+            const lastEntry = changelogs[0];
+            if (_.isUndefined(lastEntry.timestamp)) {
+                lastEntry.timestamp = TODAYS_TIMESTAMP;
+            }
+            // Check version number is correct.
+            const proposedNextVersion = lastEntry.version;
+            lastEntry.version = updateVersionNumberIfNeeded(currentVersion, proposedNextVersion);
+            changelogs[0] = lastEntry;
+            packageToVersionChange[packageName] = semverDiff(currentVersion, lastEntry.version);
+        }
+
+        // Save updated CHANGELOG.json
+        fs.writeFileSync(changelogJSONPath, JSON.stringify(changelogs, null, 4));
+        await utils.prettifyAsync(changelogJSONPath, constants.monorepoRootPath);
+        utils.log(`${packageName}: Updated CHANGELOG.json`);
+        // Generate updated CHANGELOG.md
+        const changelogMd = generateChangelogMd(changelogs);
+        const changelogMdPath = path.join(lernaPackage.location, 'CHANGELOG.md');
+        fs.writeFileSync(changelogMdPath, changelogMd);
+        await utils.prettifyAsync(changelogMdPath, constants.monorepoRootPath);
+        utils.log(`${packageName}: Updated CHANGELOG.md`);
+    }
+
+    if (!IS_DRY_RUN) {
+        await execAsync(`git add . --all`, { cwd: constants.monorepoRootPath });
+        await execAsync(`git commit -m "Updated CHANGELOGS"`, { cwd: constants.monorepoRootPath });
+        await execAsync(`git push`, { cwd: constants.monorepoRootPath });
+        utils.log(`Pushed CHANGELOG updates to Github`);
+    }
+
+    utils.log('Version updates to apply:');
+    _.each(packageToVersionChange, (versionChange: string, packageName: string) => {
+        utils.log(`${packageName} -> ${versionChange}`);
+    });
+    utils.log(`Calling 'lerna publish'...`);
+    await lernaPublishAsync(packageToVersionChange);
+})().catch(err => {
+    utils.log(err);
+    process.exit(1);
+});
+
+async function lernaPublishAsync(packageToVersionChange: { [name: string]: string }) {
+    // HACK: Lerna publish does not provide a way to specify multiple package versions via
+    // flags so instead we need to interact with their interactive prompt interface.
+    const child = spawn('lerna', ['publish', '--registry=https://registry.npmjs.org/'], {
+        cwd: constants.monorepoRootPath,
+    });
+    child.stdout.on('data', (data: Buffer) => {
+        const output = data.toString('utf8');
+        const isVersionPrompt = _.includes(output, 'Select a new version');
+        if (isVersionPrompt) {
+            const outputStripLeft = output.split('new version for ')[1];
+            const packageName = outputStripLeft.split(' ')[0];
+            let versionChange = packageToVersionChange[packageName];
+            const isPrivatePackage = _.isUndefined(versionChange);
+            if (isPrivatePackage) {
+                versionChange = 'patch'; // Always patch updates to private packages.
+            }
+            const semVerIndex = semverNameToIndex[versionChange];
+            child.stdin.write(`${semVerIndex}\n`);
+        }
+        const isFinalPrompt = _.includes(output, 'Are you sure you want to publish the above changes?');
+        if (isFinalPrompt && !IS_DRY_RUN) {
+            child.stdin.write(`y\n`);
+        } else if (isFinalPrompt && IS_DRY_RUN) {
+            utils.log(
+                `Submitted all versions to Lerna but since this is a dry run, did not confirm. You need to CTRL-C to exit.`,
+            );
+        }
+    });
+}
+
+async function getPublicLernaUpdatedPackagesAsync(): Promise<UpdatedPackage[]> {
+    const result = await execAsync(`${LERNA_EXECUTABLE} updated --json`, { cwd: constants.monorepoRootPath });
+    const updatedPackages = JSON.parse(result.stdout);
+    const updatedPublicPackages = _.filter(updatedPackages, updatedPackage => !updatedPackage.private);
+    return updatedPublicPackages;
+}
+
+function updateVersionNumberIfNeeded(currentVersion: string, proposedNextVersion: string) {
+    if (proposedNextVersion === currentVersion) {
+        return utils.getNextPatchVersion(currentVersion);
+    }
+    const sortedVersions = semverSort.desc([proposedNextVersion, currentVersion]);
+    if (sortedVersions[0] !== proposedNextVersion) {
+        return utils.getNextPatchVersion(currentVersion);
+    }
+    return proposedNextVersion;
+}
+
+function getChangelogJSONOrCreateIfMissing(packageName: string, changelogPath: string): string {
+    let changelogJSON: string;
+    try {
+        changelogJSON = fs.readFileSync(changelogPath, 'utf-8');
+        return changelogJSON;
+    } catch (err) {
+        // If none exists, create new, empty one.
+        const emptyChangelogJSON = JSON.stringify([], null, 4);
+        fs.writeFileSync(changelogPath, emptyChangelogJSON);
+        return emptyChangelogJSON;
+    }
+}
+
+function shouldAddNewChangelogEntry(changelogs: Changelog[]): boolean {
+    if (_.isEmpty(changelogs)) {
+        return true;
+    }
+    const lastEntry = changelogs[0];
+    return !!lastEntry.isPublished;
+}
+
+function generateChangelogMd(changelogs: Changelog[]): string {
+    let changelogMd = `<!--
+This file is auto-generated using the monorepo-scripts package. Don't edit directly.
+Edit the package's CHANGELOG.json file only.
+-->
+
+CHANGELOG
+    `;
+
+    _.each(changelogs, changelog => {
+        if (_.isUndefined(changelog.timestamp)) {
+            throw new Error(
+                'All CHANGELOG.json entries must be updated to include a timestamp before generating their MD version',
+            );
+        }
+        const date = moment(`${changelog.timestamp}`, 'X').format('MMMM D, YYYY');
+        const title = `\n## v${changelog.version} - _${date}_\n\n`;
+        changelogMd += title;
+
+        let changes = '';
+        _.each(changelog.changes, change => {
+            let line = `    * ${change.note}`;
+            if (!_.isUndefined(change.pr)) {
+                line += ` (#${change.pr})`;
+            }
+            line += '\n';
+            changes += line;
+        });
+        changelogMd += `${changes}`;
+    });
+
+    return changelogMd;
+}
diff --git a/packages/monorepo-scripts/src/types.ts b/packages/monorepo-scripts/src/types.ts
new file mode 100644
index 000000000..7adec202f
--- /dev/null
+++ b/packages/monorepo-scripts/src/types.ts
@@ -0,0 +1,24 @@
+export interface UpdatedPackage {
+    name: string;
+    version: string;
+    private: boolean;
+}
+
+export interface Changes {
+    note: string;
+    pr?: number;
+}
+
+export interface Changelog {
+    timestamp?: number;
+    version: string;
+    changes: Changes[];
+    isPublished?: boolean;
+}
+
+export enum SemVerIndex {
+    Invalid,
+    Patch,
+    Minor,
+    Major,
+}
diff --git a/packages/monorepo-scripts/src/utils.ts b/packages/monorepo-scripts/src/utils.ts
index 5423cabd9..9aa37e272 100644
--- a/packages/monorepo-scripts/src/utils.ts
+++ b/packages/monorepo-scripts/src/utils.ts
@@ -1,5 +1,20 @@
+import * as _ from 'lodash';
+import { exec as execAsync, spawn } from 'promisify-child-process';
+
 export const utils = {
     log(...args: any[]): void {
         console.log(...args); // tslint:disable-line:no-console
     },
+    getNextPatchVersion(currentVersion: string): string {
+        const versionSegments = currentVersion.split('.');
+        const patch = _.parseInt(_.last(versionSegments) as string);
+        const newPatch = patch + 1;
+        const newPatchVersion = `${versionSegments[0]}.${versionSegments[1]}.${newPatch}`;
+        return newPatchVersion;
+    },
+    async prettifyAsync(filePath: string, cwd: string) {
+        await execAsync(`prettier --write ${filePath} --config .prettierrc`, {
+            cwd,
+        });
+    },
 };
diff --git a/packages/website/CHANGELOG.md b/packages/website/CHANGELOG.md
deleted file mode 100644
index 642b972a2..000000000
--- a/packages/website/CHANGELOG.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# CHANGELOG
-
-## v0.0.1 - _February 16, 2018_
-
-    * Re-designed wiki + doc page template (#405)
diff --git a/packages/website/package.json b/packages/website/package.json
index 3869fb040..27f76cff4 100644
--- a/packages/website/package.json
+++ b/packages/website/package.json
@@ -38,7 +38,7 @@
         "less": "^2.7.2",
         "lodash": "^4.17.4",
         "material-ui": "^0.17.1",
-        "moment": "^2.18.1",
+        "moment": "2.21.0",
         "query-string": "^5.0.1",
         "react": "15.6.1",
         "react-copy-to-clipboard": "^4.2.3",
@@ -73,7 +73,6 @@
         "@types/jsonschema": "^1.1.1",
         "@types/lodash": "4.14.104",
         "@types/material-ui": "0.18.0",
-        "@types/moment": "^2.13.0",
         "@types/node": "^8.0.53",
         "@types/query-string": "^5.0.1",
         "@types/react": "^16.0.34",