Merge branch 'master' into nm

19 files changed, 516 insertions, 241 deletions

diff --git a/app/scripts/background.js b/app/scripts/background.js
index f077ca7a8..1b96d68b5 100644
--- a/app/scripts/background.js
+++ b/app/scripts/background.js
@@ -1,6 +1,8 @@
 const urlUtil = require('url')
 const endOfStream = require('end-of-stream')
 const pipe = require('pump')
+const log = require('loglevel')
+const extension = require('extensionizer')
 const LocalStorageStore = require('obs-store/lib/localStorage')
 const storeTransform = require('obs-store/lib/transform')
 const ExtensionPlatform = require('./platforms/extension')
@@ -9,13 +11,11 @@ const migrations = require('./migrations/')
 const PortStream = require('./lib/port-stream.js')
 const NotificationManager = require('./lib/notification-manager.js')
 const MetamaskController = require('./metamask-controller')
-const extension = require('extensionizer')
 const firstTimeState = require('./first-time-state')
 
 const STORAGE_KEY = 'metamask-config'
 const METAMASK_DEBUG = 'GULP_METAMASK_DEBUG'
 
-const log = require('loglevel')
 window.log = log
 log.setDefaultLevel(METAMASK_DEBUG ? 'debug' : 'warn')
 
@@ -29,12 +29,12 @@ let popupIsOpen = false
 const diskStore = new LocalStorageStore({ storageKey: STORAGE_KEY })
 
 // initialization flow
-initialize().catch(console.error)
+initialize().catch(log.error)
 
 async function initialize () {
   const initState = await loadStateFromPersistence()
   await setupController(initState)
-  console.log('MetaMask initialization complete.')
+  log.debug('MetaMask initialization complete.')
 }
 
 //
diff --git a/app/scripts/contentscript.js b/app/scripts/contentscript.js
index acacf5d4c..90a0f1f22 100644
--- a/app/scripts/contentscript.js
+++ b/app/scripts/contentscript.js
@@ -1,11 +1,12 @@
+const fs = require('fs')
+const path = require('path')
+const pump = require('pump')
 const LocalMessageDuplexStream = require('post-message-stream')
 const PongStream = require('ping-pong-stream/pong')
-const PortStream = require('./lib/port-stream.js')
-const ObjectMultiplex = require('./lib/obj-multiplex')
+const ObjectMultiplex = require('obj-multiplex')
 const extension = require('extensionizer')
+const PortStream = require('./lib/port-stream.js')
 
-const fs = require('fs')
-const path = require('path')
 const inpageText = fs.readFileSync(path.join(__dirname, 'inpage.js')).toString()
 
 // Eventually this streaming injection could be replaced with:
@@ -50,22 +51,42 @@ function setupStreams () {
   pageStream.pipe(pluginStream).pipe(pageStream)
 
   // setup local multistream channels
-  const mx = ObjectMultiplex()
-  mx.on('error', console.error)
-  mx.pipe(pageStream).pipe(mx)
-  mx.pipe(pluginStream).pipe(mx)
+  const mux = new ObjectMultiplex()
+  pump(
+    mux,
+    pageStream,
+    mux,
+    (err) => logStreamDisconnectWarning('MetaMask Inpage', err)
+  )
+  pump(
+    mux,
+    pluginStream,
+    mux,
+    (err) => logStreamDisconnectWarning('MetaMask Background', err)
+  )
 
   // connect ping stream
   const pongStream = new PongStream({ objectMode: true })
-  pongStream.pipe(mx.createStream('pingpong')).pipe(pongStream)
+  pump(
+    mux,
+    pongStream,
+    mux,
+    (err) => logStreamDisconnectWarning('MetaMask PingPongStream', err)
+  )
 
   // connect phishing warning stream
-  const phishingStream = mx.createStream('phishing')
+  const phishingStream = mux.createStream('phishing')
   phishingStream.once('data', redirectToPhishingWarning)
 
   // ignore unused channels (handled by background, inpage)
-  mx.ignoreStream('provider')
-  mx.ignoreStream('publicConfig')
+  mux.ignoreStream('provider')
+  mux.ignoreStream('publicConfig')
+}
+
+function logStreamDisconnectWarning (remoteLabel, err) {
+  let warningMsg = `MetamaskContentscript - lost connection to ${remoteLabel}`
+  if (err) warningMsg += '\n' + err.stack
+  console.warn(warningMsg)
 }
 
 function shouldInjectWeb3 () {
diff --git a/app/scripts/controllers/transactions.js b/app/scripts/controllers/transactions.js
index 58c468e22..fb3be6073 100644
--- a/app/scripts/controllers/transactions.js
+++ b/app/scripts/controllers/transactions.js
@@ -1,6 +1,5 @@
 const EventEmitter = require('events')
 const extend = require('xtend')
-const clone = require('clone')
 const ObservableStore = require('obs-store')
 const ethUtil = require('ethereumjs-util')
 const EthQuery = require('ethjs-query')
@@ -8,6 +7,7 @@ const TxProviderUtil = require('../lib/tx-utils')
 const PendingTransactionTracker = require('../lib/pending-tx-tracker')
 const createId = require('../lib/random-id')
 const NonceTracker = require('../lib/nonce-tracker')
+const txStateHistoryHelper = require('../lib/tx-state-history-helper')
 
 module.exports = class TransactionController extends EventEmitter {
   constructor (opts) {
@@ -33,6 +33,17 @@ module.exports = class TransactionController extends EventEmitter {
           err: undefined,
         })
       },
+      getConfirmedTransactions: (address) => {
+        return this.getFilteredTxList({
+          from: address,
+          status: 'confirmed',
+          err: undefined,
+        })
+      },
+      giveUpOnTransaction: (txId) => {
+        const msg = `Gave up submitting after 3500 blocks un-mined.`
+        this.setTxStatusFailed(txId, msg)
+      },
     })
     this.query = new EthQuery(this.provider)
     this.txProviderUtil = new TxProviderUtil(this.provider)
@@ -128,19 +139,17 @@ module.exports = class TransactionController extends EventEmitter {
 
   updateTx (txMeta) {
     // create txMeta snapshot for history
-    const txMetaForHistory = clone(txMeta)
-    // dont include previous history in this snapshot
-    delete txMetaForHistory.history
-    // add snapshot to tx history
-    if (!txMeta.history) txMeta.history = []
-    txMeta.history.push(txMetaForHistory)
-
+    const currentState = txStateHistoryHelper.snapshotFromTxMeta(txMeta)
+    // recover previous tx state obj
+    const previousState = txStateHistoryHelper.replayHistory(txMeta.history)
+    // generate history entry and add to history
+    const entry = txStateHistoryHelper.generateHistoryEntry(previousState, currentState)
+    txMeta.history.push(entry)
+
+    // commit txMeta to state
     const txId = txMeta.id
     const txList = this.getFullTxList()
     const index = txList.findIndex(txData => txData.id === txId)
-    if (!txMeta.history) txMeta.history = []
-    txMeta.history.push(txMetaForHistory)
-
     txList[index] = txMeta
     this._saveTxList(txList)
     this.emit('update')
@@ -148,16 +157,22 @@ module.exports = class TransactionController extends EventEmitter {
 
   // Adds a tx to the txlist
   addTx (txMeta) {
-    const txCount = this.getTxCount()
-    const network = this.getNetwork()
-    const fullTxList = this.getFullTxList()
-    const txHistoryLimit = this.txHistoryLimit
+    // initialize history
+    txMeta.history = []
+    // capture initial snapshot of txMeta for history
+    const snapshot = txStateHistoryHelper.snapshotFromTxMeta(txMeta)
+    txMeta.history.push(snapshot)
 
     // checks if the length of the tx history is
     // longer then desired persistence limit
     // and then if it is removes only confirmed
     // or rejected tx's.
     // not tx's that are pending or unapproved
+    const txCount = this.getTxCount()
+    const network = this.getNetwork()
+    const fullTxList = this.getFullTxList()
+    const txHistoryLimit = this.txHistoryLimit
+
     if (txCount > txHistoryLimit - 1) {
       const index = fullTxList.findIndex((metaTx) => ((metaTx.status === 'confirmed' || metaTx.status === 'rejected') && network === txMeta.metamaskNetworkId))
       fullTxList.splice(index, 1)
@@ -206,7 +221,6 @@ module.exports = class TransactionController extends EventEmitter {
       status: 'unapproved',
       metamaskNetworkId: this.getNetwork(),
       txParams: txParams,
-      history: [],
     }
     // add default tx params
     await this.addTxDefaults(txMeta)
@@ -441,4 +455,4 @@ module.exports = class TransactionController extends EventEmitter {
     })
     this.memStore.updateState({ unapprovedTxs, selectedAddressTxList })
   }
-}
\ No newline at end of file
+}
diff --git a/app/scripts/keyring-controller.js b/app/scripts/keyring-controller.js
index 2edc8060e..fd57fac70 100644
--- a/app/scripts/keyring-controller.js
+++ b/app/scripts/keyring-controller.js
@@ -171,9 +171,9 @@ class KeyringController extends EventEmitter {
       return this.setupAccounts(checkedAccounts)
     })
     .then(() => this.persistAllKeyrings())
+    .then(() => this._updateMemStoreKeyrings())
     .then(() => this.fullUpdate())
     .then(() => {
-      this._updateMemStoreKeyrings()
       return keyring
     })
   }
@@ -208,6 +208,7 @@ class KeyringController extends EventEmitter {
     return selectedKeyring.addAccounts(1)
     .then(this.setupAccounts.bind(this))
     .then(this.persistAllKeyrings.bind(this))
+    .then(this._updateMemStoreKeyrings.bind(this))
     .then(this.fullUpdate.bind(this))
   }
 
diff --git a/app/scripts/lib/auto-reload.js b/app/scripts/lib/auto-reload.js
index 6abce73ea..cce31c3d2 100644
--- a/app/scripts/lib/auto-reload.js
+++ b/app/scripts/lib/auto-reload.js
@@ -2,33 +2,55 @@ module.exports = setupDappAutoReload
 
 function setupDappAutoReload (web3, observable) {
   // export web3 as a global, checking for usage
+  let hasBeenWarned = false
+  let reloadInProgress = false
+  let lastTimeUsed
+  let lastSeenNetwork
+
   global.web3 = new Proxy(web3, {
-    get: (_web3, name) => {
-      // get the time of use
-      if (name !== '_used') {
-        console.warn('MetaMask: web3 will be deprecated in the near future in favor of the ethereumProvider \nhttps://github.com/ethereum/mist/releases/tag/v0.9.0')
-        _web3._used = Date.now()
+    get: (_web3, key) => {
+      // show warning once on web3 access
+      if (!hasBeenWarned && key !== 'currentProvider') {
+        console.warn('MetaMask: web3 will be deprecated in the near future in favor of the ethereumProvider \nhttps://github.com/MetaMask/faq/blob/master/detecting_metamask.md#web3-deprecation')
+        hasBeenWarned = true
       }
-      return _web3[name]
+      // get the time of use
+      lastTimeUsed = Date.now()
+      // return value normally
+      return _web3[key]
     },
-    set: (_web3, name, value) => {
-      _web3[name] = value
+    set: (_web3, key, value) => {
+      // set value normally
+      _web3[key] = value
     },
   })
-  var networkVersion
 
   observable.subscribe(function (state) {
-    // get the initial network
-    const curentNetVersion = state.networkVersion
-    if (!networkVersion) networkVersion = curentNetVersion
-
-    if (curentNetVersion !== networkVersion && web3._used) {
-      const timeSinceUse = Date.now() - web3._used
-      // if web3 was recently used then delay the reloading of the page
-      timeSinceUse > 500 ? triggerReset() : setTimeout(triggerReset, 500)
-      // prevent reentry into if statement if state updates again before
-      // reload
-      networkVersion = curentNetVersion
+    // if reload in progress, no need to check reload logic
+    if (reloadInProgress) return
+
+    const currentNetwork = state.networkVersion
+
+    // set the initial network
+    if (!lastSeenNetwork) {
+      lastSeenNetwork = currentNetwork
+      return
+    }
+
+    // skip reload logic if web3 not used
+    if (!lastTimeUsed) return
+
+    // if network did not change, exit
+    if (currentNetwork === lastSeenNetwork) return
+
+    // initiate page reload
+    reloadInProgress = true
+    const timeSinceUse = Date.now() - lastTimeUsed
+    // if web3 was recently used then delay the reloading of the page
+    if (timeSinceUse > 500) {
+      triggerReset()
+    } else {
+      setTimeout(triggerReset, 500)
     }
   })
 }
diff --git a/app/scripts/lib/createLoggerMiddleware.js b/app/scripts/lib/createLoggerMiddleware.js
new file mode 100644
index 000000000..b92a965de
--- /dev/null
+++ b/app/scripts/lib/createLoggerMiddleware.js
@@ -0,0 +1,15 @@
+// log rpc activity
+module.exports = createLoggerMiddleware
+
+function createLoggerMiddleware({ origin }) {
+  return function loggerMiddleware (req, res, next, end) {
+    next((cb) => {
+      if (res.error) {
+        log.error('Error in RPC response:\n', res)
+      }
+      if (req.isMetamaskInternal) return
+      log.info(`RPC (${origin}):`, req, '->', res)
+      cb()
+    })
+  }
+}
\ No newline at end of file
diff --git a/app/scripts/lib/createOriginMiddleware.js b/app/scripts/lib/createOriginMiddleware.js
new file mode 100644
index 000000000..e1e097cc4
--- /dev/null
+++ b/app/scripts/lib/createOriginMiddleware.js
@@ -0,0 +1,9 @@
+// append dapp origin domain to request
+module.exports = createOriginMiddleware
+
+function createOriginMiddleware({ origin }) {
+  return function originMiddleware (req, res, next, end) {
+    req.origin = origin
+    next()
+  }
+}
\ No newline at end of file
diff --git a/app/scripts/lib/createProviderMiddleware.js b/app/scripts/lib/createProviderMiddleware.js
new file mode 100644
index 000000000..6dd192411
--- /dev/null
+++ b/app/scripts/lib/createProviderMiddleware.js
@@ -0,0 +1,13 @@
+
+module.exports = createProviderMiddleware
+
+// forward requests to provider
+function createProviderMiddleware({ provider }) {
+  return (req, res, next, end) => {
+    provider.sendAsync(req, (err, _res) => {
+      if (err) return end(err)
+      res.result = _res.result
+      end()
+    })
+  }
+}
\ No newline at end of file
diff --git a/app/scripts/lib/inpage-provider.js b/app/scripts/lib/inpage-provider.js
index fd032a673..da75c4be2 100644
--- a/app/scripts/lib/inpage-provider.js
+++ b/app/scripts/lib/inpage-provider.js
@@ -1,8 +1,9 @@
-const pipe = require('pump')
-const StreamProvider = require('web3-stream-provider')
+const pump = require('pump')
+const RpcEngine = require('json-rpc-engine')
+const createIdRemapMiddleware = require('json-rpc-engine/src/idRemapMiddleware')
+const createStreamMiddleware = require('json-rpc-middleware-stream')
 const LocalStorageStore = require('obs-store')
-const ObjectMultiplex = require('./obj-multiplex')
-const createRandomId = require('./random-id')
+const ObjectMultiplex = require('obj-multiplex')
 
 module.exports = MetamaskInpageProvider
 
@@ -10,60 +11,49 @@ function MetamaskInpageProvider (connectionStream) {
   const self = this
 
   // setup connectionStream multiplexing
-  var multiStream = self.multiStream = ObjectMultiplex()
-  pipe(
+  const mux = self.mux = new ObjectMultiplex()
+  pump(
     connectionStream,
-    multiStream,
+    mux,
     connectionStream,
     (err) => logStreamDisconnectWarning('MetaMask', err)
   )
 
   // subscribe to metamask public config (one-way)
   self.publicConfigStore = new LocalStorageStore({ storageKey: 'MetaMask-Config' })
-  pipe(
-    multiStream.createStream('publicConfig'),
+  pump(
+    mux.createStream('publicConfig'),
     self.publicConfigStore,
     (err) => logStreamDisconnectWarning('MetaMask PublicConfigStore', err)
   )
 
   // ignore phishing warning message (handled elsewhere)
-  multiStream.ignoreStream('phishing') 
+  mux.ignoreStream('phishing')
 
   // connect to async provider
-  const asyncProvider = self.asyncProvider = new StreamProvider()
-  pipe(
-    asyncProvider,
-    multiStream.createStream('provider'),
-    asyncProvider,
+  const streamMiddleware = createStreamMiddleware()
+  pump(
+    streamMiddleware.stream,
+    mux.createStream('provider'),
+    streamMiddleware.stream,
     (err) => logStreamDisconnectWarning('MetaMask RpcProvider', err)
   )
-  // start and stop polling to unblock first block lock
-
-  self.idMap = {}
-  // handle sendAsync requests via asyncProvider
-  self.sendAsync = function (payload, cb) {
-    // rewrite request ids
-    var request = eachJsonMessage(payload, (message) => {
-      var newId = createRandomId()
-      self.idMap[newId] = message.id
-      message.id = newId
-      return message
-    })
-    // forward to asyncProvider
-    asyncProvider.sendAsync(request, function (err, res) {
-      if (err) return cb(err)
-      // transform messages to original ids
-      eachJsonMessage(res, (message) => {
-        var oldId = self.idMap[message.id]
-        delete self.idMap[message.id]
-        message.id = oldId
-        return message
-      })
-      cb(null, res)
-    })
-  }
+
+  // handle sendAsync requests via dapp-side rpc engine
+  const rpcEngine = new RpcEngine()
+  rpcEngine.push(createIdRemapMiddleware())
+  rpcEngine.push(streamMiddleware)
+  self.rpcEngine = rpcEngine
+}
+
+// handle sendAsync requests via asyncProvider
+// also remap ids inbound and outbound
+MetamaskInpageProvider.prototype.sendAsync = function (payload, cb) {
+  const self = this
+  self.rpcEngine.handle(payload, cb)
 }
 
+
 MetamaskInpageProvider.prototype.send = function (payload) {
   const self = this
 
@@ -80,7 +70,7 @@ MetamaskInpageProvider.prototype.send = function (payload) {
     case 'eth_coinbase':
       // read from localStorage
       selectedAddress = self.publicConfigStore.getState().selectedAddress
-      result = selectedAddress
+      result = selectedAddress || null
       break
 
     case 'eth_uninstallFilter':
@@ -90,7 +80,7 @@ MetamaskInpageProvider.prototype.send = function (payload) {
 
     case 'net_version':
       const networkVersion = self.publicConfigStore.getState().networkVersion
-      result = networkVersion
+      result = networkVersion || null
       break
 
     // throw not-supported Error
@@ -109,10 +99,6 @@ MetamaskInpageProvider.prototype.send = function (payload) {
   }
 }
 
-MetamaskInpageProvider.prototype.sendAsync = function () {
-  throw new Error('MetamaskInpageProvider - sendAsync not overwritten')
-}
-
 MetamaskInpageProvider.prototype.isConnected = function () {
   return true
 }
@@ -121,14 +107,6 @@ MetamaskInpageProvider.prototype.isMetaMask = true
 
 // util
 
-function eachJsonMessage (payload, transformFn) {
-  if (Array.isArray(payload)) {
-    return payload.map(transformFn)
-  } else {
-    return transformFn(payload)
-  }
-}
-
 function logStreamDisconnectWarning (remoteLabel, err) {
   let warningMsg = `MetamaskInpageProvider - lost connection to ${remoteLabel}`
   if (err) warningMsg += '\n' + err.stack
diff --git a/app/scripts/lib/nonce-tracker.js b/app/scripts/lib/nonce-tracker.js
index 8328e81ec..0029ac953 100644
--- a/app/scripts/lib/nonce-tracker.js
+++ b/app/scripts/lib/nonce-tracker.js
@@ -1,13 +1,14 @@
-const EthQuery = require('eth-query')
+const EthQuery = require('ethjs-query')
 const assert = require('assert')
 const Mutex = require('await-semaphore').Mutex
 
 class NonceTracker {
 
-  constructor ({ provider, getPendingTransactions }) {
+  constructor ({ provider, getPendingTransactions, getConfirmedTransactions }) {
     this.provider = provider
     this.ethQuery = new EthQuery(provider)
     this.getPendingTransactions = getPendingTransactions
+    this.getConfirmedTransactions = getConfirmedTransactions
     this.lockMap = {}
   }
 
@@ -25,21 +26,28 @@ class NonceTracker {
     await this._globalMutexFree()
     // await lock free, then take lock
     const releaseLock = await this._takeMutex(address)
-    // calculate next nonce
-    // we need to make sure our base count
-    // and pending count are from the same block
-    const currentBlock = await this._getCurrentBlock()
-    const pendingTransactions = this.getPendingTransactions(address)
-    const pendingCount = pendingTransactions.length
-    assert(Number.isInteger(pendingCount), `nonce-tracker - pendingCount is not an integer - got: (${typeof pendingCount}) "${pendingCount}"`)
-    const baseCountHex = await this._getTxCount(address, currentBlock)
-    const baseCount = parseInt(baseCountHex, 16)
-    assert(Number.isInteger(baseCount), `nonce-tracker - baseCount is not an integer - got: (${typeof baseCount}) "${baseCount}"`)
-    const nextNonce = baseCount + pendingCount
+    // evaluate multiple nextNonce strategies
+    const nonceDetails = {}
+    const networkNonceResult = await this._getNetworkNextNonce(address)
+    const highestLocallyConfirmed = this._getHighestLocallyConfirmed(address)
+    const nextNetworkNonce = networkNonceResult.nonce
+    const highestLocalNonce = highestLocallyConfirmed
+    const highestSuggested = Math.max(nextNetworkNonce, highestLocalNonce)
+
+    const pendingTxs = this.getPendingTransactions(address)
+    const localNonceResult = this._getHighestContinuousFrom(pendingTxs, highestSuggested) || 0
+
+    nonceDetails.params = {
+      highestLocalNonce,
+      highestSuggested,
+      nextNetworkNonce,
+    }
+    nonceDetails.local = localNonceResult
+    nonceDetails.network = networkNonceResult
+
+    const nextNonce = Math.max(networkNonceResult.nonce, localNonceResult.nonce)
     assert(Number.isInteger(nextNonce), `nonce-tracker - nextNonce is not an integer - got: (${typeof nextNonce}) "${nextNonce}"`)
-    // collect the numbers used to calculate the nonce for debugging
-    const blockNumber = currentBlock.number
-    const nonceDetails = { blockNumber, baseCount, baseCountHex, pendingCount }
+
     // return nonce and release cb
     return { nextNonce, nonceDetails, releaseLock }
   }
@@ -53,15 +61,6 @@ class NonceTracker {
     })
   }
 
-  async _getTxCount (address, currentBlock) {
-    const blockNumber = currentBlock.number
-    return new Promise((resolve, reject) => {
-      this.ethQuery.getTransactionCount(address, blockNumber, (err, result) => {
-        err ? reject(err) : resolve(result)
-      })
-    })
-  }
-
   async _globalMutexFree () {
     const globalMutex = this._lookupMutex('global')
     const release = await globalMutex.acquire()
@@ -83,12 +82,68 @@ class NonceTracker {
     return mutex
   }
 
+  async _getNetworkNextNonce (address) {
+    // calculate next nonce
+    // we need to make sure our base count
+    // and pending count are from the same block
+    const currentBlock = await this._getCurrentBlock()
+    const blockNumber = currentBlock.blockNumber
+    const baseCountBN = await this.ethQuery.getTransactionCount(address, blockNumber || 'latest')
+    const baseCount = baseCountBN.toNumber()
+    assert(Number.isInteger(baseCount), `nonce-tracker - baseCount is not an integer - got: (${typeof baseCount}) "${baseCount}"`)
+    const nonceDetails = { blockNumber, baseCount }
+    return { name: 'network', nonce: baseCount, details: nonceDetails }
+  }
+
+  _getHighestLocallyConfirmed (address) {
+    const confirmedTransactions = this.getConfirmedTransactions(address)
+    const highest = this._getHighestNonce(confirmedTransactions)
+    return Number.isInteger(highest) ? highest + 1 : 0
+  }
+
+  _reduceTxListToUniqueNonces (txList) {
+    const reducedTxList = txList.reduce((reducedList, txMeta, index) => {
+      if (!index) return [txMeta]
+      const nonceMatches = txList.filter((txData) => {
+        return txMeta.txParams.nonce === txData.txParams.nonce
+      })
+      if (nonceMatches.length > 1) return reducedList
+      reducedList.push(txMeta)
+      return reducedList
+    }, [])
+    return reducedTxList
+  }
+
+  _getHighestNonce (txList) {
+    const nonces = txList.map((txMeta) => {
+      const nonce = txMeta.txParams.nonce
+      assert(typeof nonce, 'string', 'nonces should be hex strings')
+      return parseInt(nonce, 16)
+    })
+    const highestNonce = Math.max.apply(null, nonces)
+    return highestNonce
+  }
+
+  _getHighestContinuousFrom (txList, startPoint) {
+    const nonces = txList.map((txMeta) => {
+      const nonce = txMeta.txParams.nonce
+      assert(typeof nonce, 'string', 'nonces should be hex strings')
+      return parseInt(nonce, 16)
+    })
+
+    let highest = startPoint
+    while (nonces.includes(highest)) {
+      highest++
+    }
+
+    return { name: 'local', nonce: highest, details: { startPoint, highest } }
+  }
+
   // this is a hotfix for the fact that the blockTracker will
   // change when the network changes
   _getBlockTracker () {
     return this.provider._blockTracker
   }
-
 }
 
 module.exports = NonceTracker
diff --git a/app/scripts/lib/obj-multiplex.js b/app/scripts/lib/obj-multiplex.js
deleted file mode 100644
index 0034febe0..000000000
--- a/app/scripts/lib/obj-multiplex.js
+++ /dev/null
@@ -1,48 +0,0 @@
-const through = require('through2')
-
-module.exports = ObjectMultiplex
-
-function ObjectMultiplex (opts) {
-  opts = opts || {}
-  // create multiplexer
-  const mx = through.obj(function (chunk, enc, cb) {
-    const name = chunk.name
-    const data = chunk.data
-    if (!name) {
-      console.warn(`ObjectMultiplex - Malformed chunk without name "${chunk}"`)
-      return cb()
-    }
-    const substream = mx.streams[name]
-    if (!substream) {
-      console.warn(`ObjectMultiplex - orphaned data for stream "${name}"`)
-    } else {
-      if (substream.push) substream.push(data)
-    }
-    return cb()
-  })
-  mx.streams = {}
-  // create substreams
-  mx.createStream = function (name) {
-    const substream = mx.streams[name] = through.obj(function (chunk, enc, cb) {
-      mx.push({
-        name: name,
-        data: chunk,
-      })
-      return cb()
-    })
-    mx.on('end', function () {
-      return substream.emit('end')
-    })
-    if (opts.error) {
-      mx.on('error', function () {
-        return substream.emit('error')
-      })
-    }
-    return substream
-  }
-  // ignore streams (dont display orphaned data warning)
-  mx.ignoreStream = function (name) {
-    mx.streams[name] = true
-  }
-  return mx
-}
diff --git a/app/scripts/lib/pending-tx-tracker.js b/app/scripts/lib/pending-tx-tracker.js
index 19720db3f..b90851b58 100644
--- a/app/scripts/lib/pending-tx-tracker.js
+++ b/app/scripts/lib/pending-tx-tracker.js
@@ -1,6 +1,7 @@
 const EventEmitter = require('events')
 const EthQuery = require('ethjs-query')
 const sufficientBalance = require('./util').sufficientBalance
+const RETRY_LIMIT = 3500 // Retry 3500 blocks, or about 1 day.
 /*
 
   Utility class for tracking the transactions as they
@@ -28,6 +29,7 @@ module.exports = class PendingTransactionTracker extends EventEmitter {
     this.getBalance = config.getBalance
     this.getPendingTransactions = config.getPendingTransactions
     this.publishTransaction = config.publishTransaction
+    this.giveUpOnTransaction = config.giveUpOnTransaction
   }
 
   //  checks if a signed tx is in a block and
@@ -100,6 +102,10 @@ module.exports = class PendingTransactionTracker extends EventEmitter {
     if (balance === undefined) return
     if (!('retryCount' in txMeta)) txMeta.retryCount = 0
 
+    if (txMeta.retryCount > RETRY_LIMIT) {
+      return this.giveUpOnTransaction(txMeta.id)
+    }
+
     // if the value of the transaction is greater then the balance, fail.
     if (!sufficientBalance(txMeta.txParams, balance)) {
       const insufficientFundsError = new Error('Insufficient balance during rebroadcast.')
@@ -160,4 +166,4 @@ module.exports = class PendingTransactionTracker extends EventEmitter {
     }
     nonceGlobalLock.releaseLock()
   }
-}
\ No newline at end of file
+}
diff --git a/app/scripts/lib/port-stream.js b/app/scripts/lib/port-stream.js
index 607a9c9ed..648d88087 100644
--- a/app/scripts/lib/port-stream.js
+++ b/app/scripts/lib/port-stream.js
@@ -1,5 +1,6 @@
 const Duplex = require('readable-stream').Duplex
 const inherits = require('util').inherits
+const noop = function(){}
 
 module.exports = PortDuplexStream
 
@@ -20,20 +21,14 @@ PortDuplexStream.prototype._onMessage = function (msg) {
   if (Buffer.isBuffer(msg)) {
     delete msg._isBuffer
     var data = new Buffer(msg)
-    // console.log('PortDuplexStream - saw message as buffer', data)
     this.push(data)
   } else {
-    // console.log('PortDuplexStream - saw message', msg)
     this.push(msg)
   }
 }
 
 PortDuplexStream.prototype._onDisconnect = function () {
-  try {
-    this.push(null)
-  } catch (err) {
-    this.emit('error', err)
-  }
+  this.destroy()
 }
 
 // stream plumbing
@@ -45,19 +40,12 @@ PortDuplexStream.prototype._write = function (msg, encoding, cb) {
     if (Buffer.isBuffer(msg)) {
       var data = msg.toJSON()
       data._isBuffer = true
-      // console.log('PortDuplexStream - sent message as buffer', data)
       this._port.postMessage(data)
     } else {
-      // console.log('PortDuplexStream - sent message', msg)
       this._port.postMessage(msg)
     }
   } catch (err) {
-    // console.error(err)
     return cb(new Error('PortDuplexStream - disconnected'))
   }
   cb()
 }
-
-// util
-
-function noop () {}
diff --git a/app/scripts/lib/stream-utils.js b/app/scripts/lib/stream-utils.js
index ba79990cc..8bb0b4f3c 100644
--- a/app/scripts/lib/stream-utils.js
+++ b/app/scripts/lib/stream-utils.js
@@ -1,6 +1,6 @@
 const Through = require('through2')
-const endOfStream = require('end-of-stream')
-const ObjectMultiplex = require('./obj-multiplex')
+const ObjectMultiplex = require('obj-multiplex')
+const pump = require('pump')
 
 module.exports = {
   jsonParseStream: jsonParseStream,
@@ -23,14 +23,14 @@ function jsonStringifyStream () {
 }
 
 function setupMultiplex (connectionStream) {
-  var mx = ObjectMultiplex()
-  connectionStream.pipe(mx).pipe(connectionStream)
-  endOfStream(mx, function (err) {
-    if (err) console.error(err)
-  })
-  endOfStream(connectionStream, function (err) {
-    if (err) console.error(err)
-    mx.destroy()
-  })
-  return mx
+  const mux = new ObjectMultiplex()
+  pump(
+    connectionStream,
+    mux,
+    connectionStream,
+    (err) => {
+      if (err) console.error(err)
+    }
+  )
+  return mux
 }
diff --git a/app/scripts/lib/tx-state-history-helper.js b/app/scripts/lib/tx-state-history-helper.js
new file mode 100644
index 000000000..304069d57
--- /dev/null
+++ b/app/scripts/lib/tx-state-history-helper.js
@@ -0,0 +1,37 @@
+const jsonDiffer = require('fast-json-patch')
+const clone = require('clone')
+
+module.exports = {
+  generateHistoryEntry,
+  replayHistory,
+  snapshotFromTxMeta,
+  migrateFromSnapshotsToDiffs,
+}
+
+
+function migrateFromSnapshotsToDiffs(longHistory) {
+  return (
+    longHistory
+    // convert non-initial history entries into diffs
+    .map((entry, index) => {
+      if (index === 0) return entry
+      return generateHistoryEntry(longHistory[index - 1], entry)
+    })
+  )
+}
+
+function generateHistoryEntry(previousState, newState) {
+  return jsonDiffer.compare(previousState, newState)
+}
+
+function replayHistory(shortHistory) {
+  return shortHistory.reduce((val, entry) => jsonDiffer.applyPatch(val, entry).newDocument)
+}
+
+function snapshotFromTxMeta(txMeta) {
+  // create txMeta snapshot for history
+  const snapshot = clone(txMeta)
+  // dont include previous history in this snapshot
+  delete snapshot.history
+  return snapshot
+}
\ No newline at end of file
diff --git a/app/scripts/metamask-controller.js b/app/scripts/metamask-controller.js
index a007d6fc5..fef16c3a9 100644
--- a/app/scripts/metamask-controller.js
+++ b/app/scripts/metamask-controller.js
@@ -1,12 +1,18 @@
 const EventEmitter = require('events')
 const extend = require('xtend')
 const promiseToCallback = require('promise-to-callback')
-const pipe = require('pump')
+const pump = require('pump')
 const Dnode = require('dnode')
 const ObservableStore = require('obs-store')
 const EthStore = require('./lib/eth-store')
 const EthQuery = require('eth-query')
-const streamIntoProvider = require('web3-stream-provider/handler')
+const RpcEngine = require('json-rpc-engine')
+const debounce = require('debounce')
+const createEngineStream = require('json-rpc-middleware-stream/engineStream')
+const createFilterMiddleware = require('eth-json-rpc-filters')
+const createOriginMiddleware = require('./lib/createOriginMiddleware')
+const createLoggerMiddleware = require('./lib/createLoggerMiddleware')
+const createProviderMiddleware = require('./lib/createProviderMiddleware')
 const setupMultiplex = require('./lib/stream-utils.js').setupMultiplex
 const KeyringController = require('./keyring-controller')
 const NetworkController = require('./controllers/network')
@@ -24,8 +30,6 @@ const ConfigManager = require('./lib/config-manager')
 const nodeify = require('./lib/nodeify')
 const accountImporter = require('./account-import-strategies')
 const getBuyEthUrl = require('./lib/buy-eth-url')
-const debounce = require('debounce')
-
 const version = require('../manifest.json').version
 
 module.exports = class MetamaskController extends EventEmitter {
@@ -77,12 +81,13 @@ module.exports = class MetamaskController extends EventEmitter {
 
     // rpc provider
     this.provider = this.initializeProvider()
+    this.blockTracker = this.provider
 
     // eth data query tools
     this.ethQuery = new EthQuery(this.provider)
     this.ethStore = new EthStore({
       provider: this.provider,
-      blockTracker: this.provider,
+      blockTracker: this.blockTracker,
     })
 
     // key mgmt
@@ -109,7 +114,7 @@ module.exports = class MetamaskController extends EventEmitter {
       getNetwork: this.networkController.getNetworkState.bind(this),
       signTransaction: this.keyringController.signTransaction.bind(this.keyringController),
       provider: this.provider,
-      blockTracker: this.provider,
+      blockTracker: this.blockTracker,
       ethQuery: this.ethQuery,
       ethStore: this.ethStore,
     })
@@ -337,36 +342,43 @@ module.exports = class MetamaskController extends EventEmitter {
   setupUntrustedCommunication (connectionStream, originDomain) {
     // Check if new connection is blacklisted
     if (this.blacklistController.checkForPhishing(originDomain)) {
-      console.log('MetaMask - sending phishing warning for', originDomain)
+      log.debug('MetaMask - sending phishing warning for', originDomain)
       this.sendPhishingWarning(connectionStream, originDomain)
       return
     }
 
     // setup multiplexing
-    const mx = setupMultiplex(connectionStream)
+    const mux = setupMultiplex(connectionStream)
     // connect features
-    this.setupProviderConnection(mx.createStream('provider'), originDomain)
-    this.setupPublicConfig(mx.createStream('publicConfig'))
+    this.setupProviderConnection(mux.createStream('provider'), originDomain)
+    this.setupPublicConfig(mux.createStream('publicConfig'))
   }
 
   setupTrustedCommunication (connectionStream, originDomain) {
     // setup multiplexing
-    const mx = setupMultiplex(connectionStream)
+    const mux = setupMultiplex(connectionStream)
     // connect features
-    this.setupControllerConnection(mx.createStream('controller'))
-    this.setupProviderConnection(mx.createStream('provider'), originDomain)
+    this.setupControllerConnection(mux.createStream('controller'))
+    this.setupProviderConnection(mux.createStream('provider'), originDomain)
   }
 
   sendPhishingWarning (connectionStream, hostname) {
-    const mx = setupMultiplex(connectionStream)
-    const phishingStream = mx.createStream('phishing')
+    const mux = setupMultiplex(connectionStream)
+    const phishingStream = mux.createStream('phishing')
     phishingStream.write({ hostname })
   }
 
   setupControllerConnection (outStream) {
     const api = this.getApi()
     const dnode = Dnode(api)
-    outStream.pipe(dnode).pipe(outStream)
+    pump(
+      outStream,
+      dnode,
+      outStream,
+      (err) => {
+        if (err) log.error(err)
+      }
+    )
     dnode.on('remote', (remote) => {
       // push updates to popup
       const sendUpdate = remote.sendUpdate.bind(remote)
@@ -374,27 +386,42 @@ module.exports = class MetamaskController extends EventEmitter {
     })
   }
 
-  setupProviderConnection (outStream, originDomain) {
-    streamIntoProvider(outStream, this.provider, onRequest, onResponse)
-    // append dapp origin domain to request
-    function onRequest (request) {
-      request.origin = originDomain
-    }
-    // log rpc activity
-    function onResponse (err, request, response) {
-      if (err) return console.error(err)
-      if (response.error) {
-        console.error('Error in RPC response:\n', response)
+  setupProviderConnection (outStream, origin) {
+    // setup json rpc engine stack
+    const engine = new RpcEngine()
+
+    // create filter polyfill middleware
+    const filterMiddleware = createFilterMiddleware({
+      provider: this.provider,
+      blockTracker: this.blockTracker,
+    })
+
+    engine.push(createOriginMiddleware({ origin }))
+    engine.push(createLoggerMiddleware({ origin }))
+    engine.push(filterMiddleware)
+    engine.push(createProviderMiddleware({ provider: this.provider }))
+
+    // setup connection
+    const providerStream = createEngineStream({ engine })
+    pump(
+      outStream,
+      providerStream,
+      outStream,
+      (err) => {
+        // cleanup filter polyfill middleware
+        filterMiddleware.destroy()
+        if (err) log.error(err)
       }
-      if (request.isMetamaskInternal) return
-      log.info(`RPC (${originDomain}):`, request, '->', response)
-    }
+    )
   }
 
   setupPublicConfig (outStream) {
-    pipe(
+    pump(
       this.publicConfigStore,
-      outStream
+      outStream,
+      (err) => {
+        if (err) log.error(err)
+      }
     )
   }
 
diff --git a/app/scripts/migrations/018.js b/app/scripts/migrations/018.js
new file mode 100644
index 000000000..d27fe3f46
--- /dev/null
+++ b/app/scripts/migrations/018.js
@@ -0,0 +1,52 @@
+const version = 18
+
+/*
+
+This migration updates "transaction state history" to diffs style
+
+*/
+
+const clone = require('clone')
+const txStateHistoryHelper = require('../lib/tx-state-history-helper')
+
+
+module.exports = {
+  version,
+
+  migrate: function (originalVersionedData) {
+    const versionedData = clone(originalVersionedData)
+    versionedData.meta.version = version
+    try {
+      const state = versionedData.data
+      const newState = transformState(state)
+      versionedData.data = newState
+    } catch (err) {
+      console.warn(`MetaMask Migration #${version}` + err.stack)
+    }
+    return Promise.resolve(versionedData)
+  },
+}
+
+function transformState (state) {
+  const newState = state
+  const transactions = newState.TransactionController.transactions
+  newState.TransactionController.transactions = transactions.map((txMeta) => {
+    // no history: initialize
+    if (!txMeta.history || txMeta.history.length === 0) {
+      const snapshot = txStateHistoryHelper.snapshotFromTxMeta(txMeta)
+      txMeta.history = [snapshot]
+      return txMeta
+    }
+    // has history: migrate
+    const newHistory = (
+      txStateHistoryHelper.migrateFromSnapshotsToDiffs(txMeta.history)
+      // remove empty diffs
+      .filter((entry) => {
+        return !Array.isArray(entry) || entry.length > 0
+      })
+    )
+    txMeta.history = newHistory
+    return txMeta
+  })
+  return newState
+}
diff --git a/app/scripts/migrations/019.js b/app/scripts/migrations/019.js
new file mode 100644
index 000000000..072c96370
--- /dev/null
+++ b/app/scripts/migrations/019.js
@@ -0,0 +1,83 @@
+
+const version = 19
+
+/*
+
+This migration sets transactions as failed
+whos nonce is too high
+
+*/
+
+const clone = require('clone')
+
+module.exports = {
+  version,
+
+  migrate: function (originalVersionedData) {
+    const versionedData = clone(originalVersionedData)
+    versionedData.meta.version = version
+    try {
+      const state = versionedData.data
+      const newState = transformState(state)
+      versionedData.data = newState
+    } catch (err) {
+      console.warn(`MetaMask Migration #${version}` + err.stack)
+    }
+    return Promise.resolve(versionedData)
+  },
+}
+
+function transformState (state) {
+  const newState = state
+  const transactions = newState.TransactionController.transactions
+
+  newState.TransactionController.transactions = transactions.map((txMeta, _, txList) => {
+    if (txMeta.status !== 'submitted') return txMeta
+
+    const confirmedTxs = txList.filter((tx) => tx.status === 'confirmed')
+    .filter((tx) => tx.txParams.from === txMeta.txParams.from)
+    .filter((tx) => tx.metamaskNetworkId.from === txMeta.metamaskNetworkId.from)
+    const highestConfirmedNonce = getHighestNonce(confirmedTxs)
+
+    const pendingTxs = txList.filter((tx) => tx.status === 'submitted')
+    .filter((tx) => tx.txParams.from === txMeta.txParams.from)
+    .filter((tx) => tx.metamaskNetworkId.from === txMeta.metamaskNetworkId.from)
+    const highestContinuousNonce = getHighestContinuousFrom(pendingTxs, highestConfirmedNonce)
+
+    const maxNonce = Math.max(highestContinuousNonce, highestConfirmedNonce)
+
+    if (parseInt(txMeta.txParams.nonce, 16) > maxNonce + 1) {
+      txMeta.status = 'failed'
+      txMeta.err = {
+        message: 'nonce too high',
+        note: 'migration 019 custom error',
+      }
+    }
+    return txMeta
+  })
+  return newState
+}
+
+function getHighestContinuousFrom (txList, startPoint) {
+  const nonces = txList.map((txMeta) => {
+    const nonce = txMeta.txParams.nonce
+    return parseInt(nonce, 16)
+  })
+
+  let highest = startPoint
+  while (nonces.includes(highest)) {
+    highest++
+  }
+
+  return highest
+}
+
+function getHighestNonce (txList) {
+  const nonces = txList.map((txMeta) => {
+  const nonce = txMeta.txParams.nonce
+    return parseInt(nonce || '0x0', 16)
+  })
+  const highestNonce = Math.max.apply(null, nonces)
+  return highestNonce
+}
+
diff --git a/app/scripts/migrations/index.js b/app/scripts/migrations/index.js
index f4c87499f..e9cbd7b98 100644
--- a/app/scripts/migrations/index.js
+++ b/app/scripts/migrations/index.js
@@ -28,4 +28,6 @@ module.exports = [
   require('./015'),
   require('./016'),
   require('./017'),
+  require('./018'),
+  require('./019'),
 ]