build: add ci.go, use it everywhere

The new build script, ci.go, replaces some of the older shell scripts.
ci.go can compile go-ethereum, run the tests, create release archives
and debian source packages.

12 files changed, 552 insertions, 59 deletions

diff --git a/build/ci-notes.md b/build/ci-notes.md
new file mode 100644
index 000000000..989cba6dd
--- /dev/null
+++ b/build/ci-notes.md
@@ -0,0 +1,26 @@
+Debian Packaging
+----------------
+
+Tagged releases and develop branch commits are available as installable Debian packages
+for Ubuntu. Packages are built for the all Ubuntu versions which are supported by
+Canonical:
+
+- Trusty Tahr (14.04 LTS)
+- Wily Werewolf (15.10)
+- Xenial Xerus (16.04 LTS)
+
+Packages of develop branch commits have suffix -unstable and cannot be installed alongside
+the stable version. Switching between release streams requires user intervention.
+
+The packages are built and served by launchpad.net. We generate a Debian source package
+for each distribution and upload it. Their builder picks up the source package, builds it
+and installs the new version into the PPA repository. Launchpad requires a valid signature
+by a team member for source package uploads. The signing key is stored in an environment
+variable which Travis CI makes available to certain builds.
+
+We want to build go-ethereum with the most recent version of Go, irrespective of the Go
+version that is available in the main Ubuntu repository. In order to make this possible,
+our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
+golang-1.6, which is co-installable alongside the regular golang package. PPA dependencies
+can be edited at https://launchpad.net/%7Elp-fjl/+archive/ubuntu/geth-ci-testing/+edit-dependencies
+
diff --git a/build/ci.go b/build/ci.go
new file mode 100644
index 000000000..33d97c182
--- /dev/null
+++ b/build/ci.go
@@ -0,0 +1,465 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build none
+
+/*
+The ci command is called from Continuous Integration scripts.
+
+Usage: go run ci.go <command> <command flags/arguments>
+
+Available commands are:
+
+   install    [ packages... ]                          -- builds packages and executables
+   test       [ -coverage ] [ -vet ] [ packages... ]   -- runs the tests
+   archive    [ -type zip|tar ]                        -- archives build artefacts
+   importkeys                                          -- imports signing keys from env
+   debsrc     [ -sign key-id ] [ -upload dest ]        -- creates a debian source package
+
+For all commands, -n prevents execution of external programs (dry run mode).
+
+*/
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"../internal/build"
+)
+
+var (
+	// Files that end up in the geth*.zip archive.
+	gethArchiveFiles = []string{
+		"COPYING",
+		executablePath("geth"),
+	}
+
+	// Files that end up in the geth-alltools*.zip archive.
+	allToolsArchiveFiles = []string{
+		"COPYING",
+		executablePath("abigen"),
+		executablePath("evm"),
+		executablePath("geth"),
+		executablePath("rlpdump"),
+	}
+
+	// A debian package is created for all executables listed here.
+	debExecutables = []debExecutable{
+		{
+			Name:        "geth",
+			Description: "Ethereum CLI client.",
+		},
+		{
+			Name:        "rlpdump",
+			Description: "Developer utility tool that prints RLP structures.",
+		},
+		{
+			Name:        "evm",
+			Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
+		},
+		{
+			Name:        "abigen",
+			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
+		},
+	}
+
+	// Distros for which packages are created.
+	// Note: vivid is unsupported because there is no golang-1.6 package for it.
+	debDistros = []string{"trusty", "wily", "xenial", "yakkety"}
+)
+
+var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
+
+func executablePath(name string) string {
+	if runtime.GOOS == "windows" {
+		name += ".exe"
+	}
+	return filepath.Join(GOBIN, name)
+}
+
+func main() {
+	log.SetFlags(log.Lshortfile)
+
+	if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
+		log.Fatal("this script must be run from the root of the repository")
+	}
+	if len(os.Args) < 2 {
+		log.Fatal("need subcommand as first argument")
+	}
+	switch os.Args[1] {
+	case "install":
+		doInstall(os.Args[2:])
+	case "test":
+		doTest(os.Args[2:])
+	case "archive":
+		doArchive(os.Args[2:])
+	case "debsrc":
+		doDebianSource(os.Args[2:])
+	case "travis-debsrc":
+		doTravisDebianSource(os.Args[2:])
+	default:
+		log.Fatal("unknown command ", os.Args[1])
+	}
+}
+
+// Compiling
+
+func doInstall(cmdline []string) {
+	commitHash := flag.String("gitcommit", "", "Git commit hash embedded into binary.")
+	flag.CommandLine.Parse(cmdline)
+
+	// Check Go version. People regularly open issues about compilation
+	// failure with outdated Go. This should save them the trouble.
+	if runtime.Version() < "go1.4" && !strings.HasPrefix(runtime.Version(), "devel") {
+		log.Println("You have Go version", runtime.Version())
+		log.Println("go-ethereum requires at least Go version 1.4 and cannot")
+		log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
+		os.Exit(1)
+	}
+
+	// Compile packages given as arguments, or everything if there are no arguments.
+	packages := []string{"./..."}
+	if flag.NArg() > 0 {
+		packages = flag.Args()
+	}
+
+	goinstall := goTool("install", makeBuildFlags(*commitHash)...)
+	goinstall.Args = append(goinstall.Args, "-v")
+	goinstall.Args = append(goinstall.Args, packages...)
+	build.MustRun(goinstall)
+}
+
+func makeBuildFlags(commitHash string) (flags []string) {
+	// Since Go 1.5, the separator char for link time assignments
+	// is '=' and using ' ' prints a warning. However, Go < 1.5 does
+	// not support using '='.
+	sep := " "
+	if runtime.Version() > "go1.5" || strings.Contains(runtime.Version(), "devel") {
+		sep = "="
+	}
+
+	if os.Getenv("GO_OPENCL") != "" {
+		flags = append(flags, "-tags", "opencl")
+	}
+
+	// Set gitCommit constant via link-time assignment. If this is a git checkout, we can
+	// just get the current commit hash through git. Otherwise we fall back to the hash
+	// that was passed as -gitcommit.
+	//
+	// -gitcommit is required for Debian package builds. The source package doesn't
+	// contain .git but we still want to embed the commit hash into the packaged binary.
+	// The hash is rendered into the debian/rules build script when the source package is
+	// created.
+	if _, err := os.Stat(filepath.Join(".git", "HEAD")); !os.IsNotExist(err) {
+		if c := build.GitCommit(); c != "" {
+			commitHash = c
+		}
+	}
+	if commitHash != "" {
+		flags = append(flags, "-ldflags", "-X main.gitCommit"+sep+commitHash)
+	}
+	return flags
+}
+
+func goTool(subcmd string, args ...string) *exec.Cmd {
+	gocmd := filepath.Join(runtime.GOROOT(), "bin", "go")
+	cmd := exec.Command(gocmd, subcmd)
+	cmd.Args = append(cmd.Args, args...)
+	cmd.Env = []string{
+		"GOPATH=" + build.GOPATH(),
+		"GOBIN=" + GOBIN,
+	}
+	for _, e := range os.Environ() {
+		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
+			continue
+		}
+		cmd.Env = append(cmd.Env, e)
+	}
+	return cmd
+}
+
+// Running The Tests
+//
+// "tests" also includes static analysis tools such as vet.
+
+func doTest(cmdline []string) {
+	var (
+		vet      = flag.Bool("vet", false, "Whether to run go vet")
+		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
+	)
+	flag.CommandLine.Parse(cmdline)
+	packages := []string{"./..."}
+	if len(flag.CommandLine.Args()) > 0 {
+		packages = flag.CommandLine.Args()
+	}
+
+	// Run analysis tools before the tests.
+	if *vet {
+		build.MustRun(goTool("vet", packages...))
+	}
+
+	// Run the actual tests.
+	gotest := goTool("test")
+	if *coverage {
+		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
+	}
+	gotest.Args = append(gotest.Args, packages...)
+	build.MustRun(gotest)
+}
+
+// Release Packaging
+
+func doArchive(cmdline []string) {
+	var (
+		atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
+		ext   string
+	)
+	flag.CommandLine.Parse(cmdline)
+	switch *atype {
+	case "zip":
+		ext = ".zip"
+	case "tar":
+		ext = ".tar.gz"
+	default:
+		log.Fatal("unknown archive type: ", atype)
+	}
+	base := makeArchiveBasename()
+	if err := build.WriteArchive("geth-"+base, ext, gethArchiveFiles); err != nil {
+		log.Fatal(err)
+	}
+	if err := build.WriteArchive("geth-alltools-"+base, ext, allToolsArchiveFiles); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func makeArchiveBasename() string {
+	// date := time.Now().UTC().Format("200601021504")
+	platform := runtime.GOOS + "-" + runtime.GOARCH
+	archive := platform + "-" + build.VERSION()
+	if commit := build.GitCommit(); commit != "" {
+		archive += "-" + commit[:8]
+	}
+	return archive
+}
+
+// Debian Packaging
+
+// CLI entry point for Travis CI.
+func doTravisDebianSource(cmdline []string) {
+	flag.CommandLine.Parse(cmdline)
+
+	// Package only whitelisted branches.
+	switch {
+	case os.Getenv("TRAVIS_REPO_SLUG") != "ethereum/go-ethereum":
+		log.Printf("skipping because this is a fork build")
+		return
+	case os.Getenv("TRAVIS_PULL_REQUEST") != "false":
+		log.Printf("skipping because this is a PR build")
+		return
+	case os.Getenv("TRAVIS_BRANCH") != "develop" && !strings.HasPrefix(os.Getenv("TRAVIS_TAG"), "v1."):
+		log.Printf("skipping because branch %q tag %q is not on the whitelist",
+			os.Getenv("TRAVIS_BRANCH"),
+			os.Getenv("TRAVIS_TAG"))
+		return
+	}
+
+	// Import the signing key.
+	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
+		key, err := base64.StdEncoding.DecodeString(b64key)
+		if err != nil {
+			log.Fatal("invalid base64 PPA_SIGNING_KEY")
+		}
+		gpg := exec.Command("gpg", "--import")
+		gpg.Stdin = bytes.NewReader(key)
+		build.MustRun(gpg)
+	}
+
+	// Assign unstable status to non-tag builds.
+	unstable := "true"
+	if os.Getenv("TRAVIS_BRANCH") != "develop" && os.Getenv("TRAVIS_TAG") != "" {
+		unstable = "false"
+	}
+
+	doDebianSource([]string{
+		"-signer", "Felix Lange (Geth CI Testing Key) <fjl@twurst.com>",
+		"-buildnum", os.Getenv("TRAVIS_BUILD_NUMBER"),
+		"-upload", "ppa:lp-fjl/geth-ci-testing",
+		"-unstable", unstable,
+	})
+}
+
+// CLI entry point for doing packaging locally.
+func doDebianSource(cmdline []string) {
+	var (
+		signer   = flag.String("signer", "", `Signing key name, also used as package author`)
+		upload   = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+		buildnum = flag.String("buildnum", "", `Build number (included in version)`)
+		unstable = flag.Bool("unstable", false, `Use package name suffix "-unstable"`)
+		now      = time.Now()
+	)
+	flag.CommandLine.Parse(cmdline)
+
+	// Create the debian worktree in /tmp.
+	tmpdir, err := ioutil.TempDir("", "eth-deb-build-")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, distro := range debDistros {
+		meta := newDebMetadata(distro, *signer, *buildnum, *unstable, now)
+		pkgdir := stageDebianSource(tmpdir, meta)
+		debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
+		debuild.Dir = pkgdir
+		build.MustRun(debuild)
+
+		changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
+		changes = filepath.Join(tmpdir, changes)
+		if *signer != "" {
+			build.MustRunCommand("debsign", changes)
+		}
+		if *upload != "" {
+			build.MustRunCommand("dput", *upload, changes)
+		}
+	}
+}
+
+type debExecutable struct {
+	Name, Description string
+}
+
+type debMetadata struct {
+	// go-ethereum version being built. Note that this
+	// is not the debian package version. The package version
+	// is constructed by VersionString.
+	Version string
+
+	Author               string // "name <email>", also selects signing key
+	Buildnum             string // build number
+	Distro, Commit, Time string
+	Executables          []debExecutable
+	Unstable             bool
+}
+
+func newDebMetadata(distro, author, buildnum string, unstable bool, t time.Time) debMetadata {
+	if author == "" {
+		// No signing key, use default author.
+		author = "Ethereum Builds <fjl@ethereum.org>"
+	}
+	return debMetadata{
+		Unstable:    unstable,
+		Author:      author,
+		Distro:      distro,
+		Commit:      build.GitCommit(),
+		Version:     build.VERSION(),
+		Buildnum:    buildnum,
+		Time:        t.Format(time.RFC1123Z),
+		Executables: debExecutables,
+	}
+}
+
+// Name returns the name of the metapackage that depends
+// on all executable packages.
+func (meta debMetadata) Name() string {
+	if meta.Unstable {
+		return "ethereum-unstable"
+	}
+	return "ethereum"
+}
+
+// VersionString returns the debian version of the packages.
+func (meta debMetadata) VersionString() string {
+	vsn := meta.Version
+	if meta.Buildnum != "" {
+		vsn += "+build" + meta.Buildnum
+	}
+	if meta.Distro != "" {
+		vsn += "+" + meta.Distro
+	}
+	return vsn
+}
+
+// ExeList returns the list of all executable packages.
+func (meta debMetadata) ExeList() string {
+	names := make([]string, len(meta.Executables))
+	for i, e := range meta.Executables {
+		names[i] = meta.ExeName(e)
+	}
+	return strings.Join(names, ", ")
+}
+
+// ExeName returns the package name of an executable package.
+func (meta debMetadata) ExeName(exe debExecutable) string {
+	if meta.Unstable {
+		return exe.Name + "-unstable"
+	}
+	return exe.Name
+}
+
+// ExeConflicts returns the content of the Conflicts field
+// for executable packages.
+func (meta debMetadata) ExeConflicts(exe debExecutable) string {
+	if meta.Unstable {
+		// Set up the conflicts list so that the *-unstable packages
+		// cannot be installed alongside the regular version.
+		//
+		// https://www.debian.org/doc/debian-policy/ch-relationships.html
+		// is very explicit about Conflicts: and says that Breaks: should
+		// be preferred and the conflicting files should be handled via
+		// alternates. We might do this eventually but using a conflict is
+		// easier now.
+		return "ethereum, " + exe.Name
+	}
+	return ""
+}
+
+func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
+	pkg := meta.Name() + "-" + meta.VersionString()
+	pkgdir = filepath.Join(tmpdir, pkg)
+	if err := os.Mkdir(pkgdir, 0755); err != nil {
+		log.Fatal(err)
+	}
+
+	// Copy the source code.
+	build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
+
+	// Put the debian build files in place.
+	debian := filepath.Join(pkgdir, "debian")
+	build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
+	build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
+	build.Render("build/deb.control", filepath.Join(debian, "control"), 0644, meta)
+	build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
+	build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
+	build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
+	for _, exe := range meta.Executables {
+		install := filepath.Join(debian, exe.Name+".install")
+		docs := filepath.Join(debian, exe.Name+".docs")
+		build.Render("build/deb.install", install, 0644, exe)
+		build.Render("build/deb.docs", docs, 0644, exe)
+	}
+
+	return pkgdir
+}
diff --git a/build/deb.changelog b/build/deb.changelog
new file mode 100644
index 000000000..a221f5470
--- /dev/null
+++ b/build/deb.changelog
@@ -0,0 +1,5 @@
+{{.Name}} ({{.VersionString}}) {{.Distro}}; urgency=low
+
+  * git build of {{.Commit}}
+
+ -- {{.Author}}  {{.Time}}
diff --git a/build/deb.control b/build/deb.control
new file mode 100644
index 000000000..4a65c7fac
--- /dev/null
+++ b/build/deb.control
@@ -0,0 +1,25 @@
+Source: {{.Name}}
+Section: science
+Priority: extra
+Maintainer: {{.Author}}
+Build-Depends: debhelper (>= 8.0.0), golang-1.6
+Standards-Version: 3.9.5
+Homepage: https://ethereum.org
+Vcs-Git: git://github.com/ethereum/go-ethereum.git
+Vcs-Browser: https://github.com/ethereum/go-ethereum
+
+Package: {{.Name}}
+Architecture: any
+Depends: ${misc:Depends}, {{.ExeList}}
+Description: Meta-package to install geth and other tools
+ Meta-package to install geth and other tools
+ 
+{{range .Executables}}
+Package: {{$.ExeName .}}
+Conflicts: {{$.ExeConflicts .}}
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Built-Using: ${misc:Built-Using}
+Description: {{.Description}}
+ {{.Description}}
+{{end}}
diff --git a/build/deb.copyright b/build/deb.copyright
new file mode 100644
index 000000000..513be45b1
--- /dev/null
+++ b/build/deb.copyright
@@ -0,0 +1,14 @@
+Copyright 2016 The go-ethereum Authors
+
+go-ethereum is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+go-ethereum is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
diff --git a/build/deb.docs b/build/deb.docs
new file mode 100644
index 000000000..62deb0497
--- /dev/null
+++ b/build/deb.docs
@@ -0,0 +1 @@
+AUTHORS
diff --git a/build/deb.install b/build/deb.install
new file mode 100644
index 000000000..7dc76e1f5
--- /dev/null
+++ b/build/deb.install
@@ -0,0 +1 @@
+build/bin/{{.Name}} usr/bin
diff --git a/build/deb.rules b/build/deb.rules
new file mode 100644
index 000000000..3dfadb08d
--- /dev/null
+++ b/build/deb.rules
@@ -0,0 +1,13 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+override_dh_auto_build:
+	build/env.sh /usr/lib/go-1.6/bin/go run build/ci.go install -gitcommit {{.Commit}}
+
+override_dh_auto_test:
+
+%:
+	dh $@
diff --git a/build/env.sh b/build/env.sh
index 04401a3e1..c418dae44 100755
--- a/build/env.sh
+++ b/build/env.sh
@@ -20,9 +20,8 @@ fi
 
 # Set up the environment to use the workspace.
 # Also add Godeps workspace so we build using canned dependencies.
-GOPATH="$ethdir/go-ethereum/Godeps/_workspace:$workspace"
-GOBIN="$PWD/build/bin"
-export GOPATH GOBIN
+GOPATH="$workspace"
+export GOPATH
 
 # Run the command inside the workspace.
 cd "$ethdir/go-ethereum"
diff --git a/build/test-global-coverage.sh b/build/test-global-coverage.sh
deleted file mode 100755
index a51b6a9e5..000000000
--- a/build/test-global-coverage.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-echo "" > coverage.txt
-
-for d in $(find ./* -maxdepth 10 -type d -not -path "./build" -not -path "./Godeps/*" ); do
-    if ls $d/*.go &> /dev/null; then
-        go test  -coverprofile=profile.out -covermode=atomic $d
-        if [ -f profile.out ]; then
-            cat profile.out >> coverage.txt
-            echo '<<<<<< EOF' >> coverage.txt
-            rm profile.out
-        fi
-    fi
-done
diff --git a/build/win-ci-compile.bat b/build/win-ci-compile.bat
deleted file mode 100644
index 5750990bf..000000000
--- a/build/win-ci-compile.bat
+++ /dev/null
@@ -1,26 +0,0 @@
-@echo off
-if not exist .\build\win-ci-compile.bat (
-   echo This script must be run from the root of the repository.
-   exit /b
-)
-if not defined GOPATH (
-   echo GOPATH is not set.
-   exit /b
-)
-
-set GOPATH=%GOPATH%;%cd%\Godeps\_workspace
-set GOBIN=%cd%\build\bin
-
-rem set gitCommit when running from a Git checkout.
-set goLinkFlags=""
-if exist ".git\HEAD" (
-   where /q git
-   if not errorlevel 1 (
-      for /f %%h in ('git rev-parse HEAD') do (
-          set goLinkFlags="-X main.gitCommit=%%h"
-      )
-   )
-)
-
-@echo on
-go install -v -ldflags %goLinkFlags% ./...
diff --git a/build/win-ci-test.bat b/build/win-ci-test.bat
deleted file mode 100644
index 5945426db..000000000
--- a/build/win-ci-test.bat
+++ /dev/null
@@ -1,15 +0,0 @@
-@echo off
-if not exist .\build\win-ci-test.bat (
-   echo This script must be run from the root of the repository.
-   exit /b
-)
-if not defined GOPATH (
-   echo GOPATH is not set.
-   exit /b
-)
-
-set GOPATH=%GOPATH%;%cd%\Godeps\_workspace
-set GOBIN=%cd%\build\bin
-
-@echo on
-go test ./...