storeHeight & stateHeight does not match
baabeetaa opened this issue · 3 comments
baabeetaa commented
NOTE[02-01|16:24:00] ABCI Handshake module=state appHeight=210486 appHash=0A2973AF199B84C07933D58A5A89422228FF4399
NOTE[02-01|16:24:00] ABCI Replay Blocks module=state appHeight=210486 storeHeight=210488 stateHeight=210487
panic: Paniced on a Sanity Check: Expected storeHeight (210488) and stateHeight (210487) to match.
goroutine 1 [running]:
panic(0x9a58c0, 0xc4203ceec0)
/usr/local/go/src/runtime/panic.go:500 +0x1a1
github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common.PanicSanity(0x9a58c0, 0xc4203ceea0)
/root/GOPATH/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common/errors.go:26 +0xe0
github.com/tendermint/tendermint/state.(*Handshaker).ReplayBlocks(0xc4200b7500, 0xc4201898a0, 0x14, 0x14, 0x33636, 0xea4f60, 0xc4203cea50, 0x14, 0x14)
/root/GOPATH/src/github.com/tendermint/tendermint/state/execution.go:400 +0xda2
github.com/tendermint/tendermint/state.(*Handshaker).Handshake(0xc4200b7500, 0xea62a0, 0xc42007e1b0, 0x0, 0x0)
/root/GOPATH/src/github.com/tendermint/tendermint/state/execution.go:329 +0x483
github.com/tendermint/tendermint/proxy.(*multiAppConn).OnStart(0xc42007e1b0, 0xc420189320, 0x15)
/root/GOPATH/src/github.com/tendermint/tendermint/proxy/multi_app_conn.go:100 +0x206
github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common.(*BaseService).Start(0xc42007e1b0, 0x0, 0x0, 0x0)
/root/GOPATH/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/go-common/service.go:96 +0x546
github.com/tendermint/tendermint/node.NewNode(0xea6e20, 0xc420109ce0, 0xc420081ea0, 0xe99640, 0xc4201531d0, 0x0)
/root/GOPATH/src/github.com/tendermint/tendermint/node/node.go:68 +0x483
github.com/tendermint/tendermint/node.NewNodeDefault(0xea6e20, 0xc420109ce0, 0x1)
/root/GOPATH/src/github.com/tendermint/tendermint/node/node.go:51 +0xba
github.com/tendermint/tendermint/node.RunNode(0xea6e20, 0xc420109ce0)
/root/GOPATH/src/github.com/tendermint/tendermint/node/node.go:350 +0x529
main.main()
/root/GOPATH/src/github.com/tendermint/tendermint/cmd/tendermint/main.go:42 +0x373
baabeetaa commented
how to fix?
// Replay all blocks after blockHeight and ensure the result matches the current state.
func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnConsensus proxy.AppConnConsensus) error {
storeBlockHeight := h.store.Height()
stateBlockHeight := h.state.LastBlockHeight
log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)
if storeBlockHeight == 0 {
return nil
} else if storeBlockHeight < appBlockHeight {
// if the app is ahead, there's nothing we can do
return ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}
} else if storeBlockHeight == appBlockHeight {
// We ran Commit, but if we crashed before state.Save(),
// load the intermediate state and update the state.AppHash.
// NOTE: If ABCI allowed rollbacks, we could just replay the
// block even though it's been committed
stateAppHash := h.state.AppHash
lastBlockAppHash := h.store.LoadBlock(storeBlockHeight).AppHash
if bytes.Equal(stateAppHash, appHash) {
// we're all synced up
log.Debug("ABCI RelpayBlocks: Already synced")
} else if bytes.Equal(stateAppHash, lastBlockAppHash) {
// we crashed after commit and before saving state,
// so load the intermediate state and update the hash
h.state.LoadIntermediate()
h.state.AppHash = appHash
log.Debug("ABCI RelpayBlocks: Loaded intermediate state and updated state.AppHash")
} else {
PanicSanity(Fmt("Unexpected state.AppHash: state.AppHash %X; app.AppHash %X, lastBlock.AppHash %X", stateAppHash, appHash, lastBlockAppHash))
}
return nil
} else if storeBlockHeight == appBlockHeight+1 &&
storeBlockHeight == stateBlockHeight+1 {
// We crashed after saving the block
// but before Commit (both the state and app are behind),
// so just replay the block
// check that the lastBlock.AppHash matches the state apphash
block := h.store.LoadBlock(storeBlockHeight)
if !bytes.Equal(block.Header.AppHash, appHash) {
return ErrLastStateMismatch{storeBlockHeight, block.Header.AppHash, appHash}
}
blockMeta := h.store.LoadBlockMeta(storeBlockHeight)
h.nBlocks += 1
var eventCache types.Fireable // nil
// replay the latest block
return h.state.ApplyBlock(eventCache, appConnConsensus, block, blockMeta.PartsHeader, MockMempool{})
} else if storeBlockHeight != stateBlockHeight {
// unless we failed before committing or saving state (previous 2 case),
// the store and state should be at the same height!
PanicSanity(Fmt("Expected storeHeight (%d) and stateHeight (%d) to match.", storeBlockHeight, stateBlockHeight))
} else {
// store is more than one ahead,
// so app wants to replay many blocks
// replay all blocks starting with appBlockHeight+1
var eventCache types.Fireable // nil
// TODO: use stateBlockHeight instead and let the consensus state
// do the replay
var appHash []byte
for i := appBlockHeight + 1; i <= storeBlockHeight; i++ {
h.nBlocks += 1
block := h.store.LoadBlock(i)
_, err := execBlockOnProxyApp(eventCache, appConnConsensus, block)
if err != nil {
log.Warn("Error executing block on proxy app", "height", i, "err", err)
return err
}
// Commit block, get hash back
res := appConnConsensus.CommitSync()
if res.IsErr() {
log.Warn("Error in proxyAppConn.CommitSync", "error", res)
return res
}
if res.Log != "" {
log.Info("Commit.Log: " + res.Log)
}
appHash = res.Data
}
if !bytes.Equal(h.state.AppHash, appHash) {
return errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash))
}
return nil
}
return nil
}
baabeetaa commented
Need to stop TM before glogchain.
This bug is annoying, if app stop for some reason (got panic ) then the chain got corrupted. Have to synch again!
Dont know if combining TM + Glog to single binary is helpful?
faddat commented
In my mind the ideal form is
Binary One: (tm + glog)
Binary Two: (webapp)
Instead of getting our repositories all strange-ified, I also figure if we are doing the grand refactor then we could also create:
github.com/dawn-network/webfront
github.com/dawn-network/desktopqtclient
github.com/dawn-network/fluttermobile