Skip to content
Open
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Icon?
coverage.out
build/
hermes
tests/e2e/forge/forge

# IDEs and editors
.vscode/
Expand Down
5 changes: 3 additions & 2 deletions tests/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,9 @@ misbehaviour are accepted only from a launched consumer):
a resume is refused while the client is frozen;
4. governance recovers the frozen client onto a substitute and resumes the
consumer in one proposal: refused inside the client's trusting period and
rolled back, passed once it has elapsed; the consumer is served again and
nobody is punished;
rolled back, passed once it has elapsed; the consumer is served again,
nobody is punished, and the deferred punishment's clock resumes where the
pause stopped it;
5. the consumer restarts under the next revision of its chain id; the
provider's client expires and governance recovers it onto a client of the
renamed chain; the next provider block pauses the consumer for the chain
Expand Down
46 changes: 44 additions & 2 deletions tests/e2e/base_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,33 @@ func (s *baseTestSuite) cleanupStaleContainers() {
for _, name := range staleNames {
s.removeContainerByName(name)
}
// Also remove stale network
_ = s.dkrPool.Client.RemoveNetwork(s.cfg.dockerNetwork)
s.removeStaleNetwork()
}

// removeStaleNetwork removes the suite's docker network if a previous run
// left it behind, with whatever containers are still attached to it: a
// scenario's ephemeral export or validate container outlives a run killed
// while it ran, docker refuses to remove a network with an endpoint, and
// the name stays taken, so bringUpSuite would fail to create the network.
// A network that cannot be removed fails the suite here, with docker's
// reason, instead of at that creation.
func (s *baseTestSuite) removeStaleNetwork() {
var missing *docker.NoSuchNetwork
network, err := s.dkrPool.Client.NetworkInfo(s.cfg.dockerNetwork)
if errors.As(err, &missing) {
return
}
s.Require().NoErrorf(err, "failed to inspect the docker network %s", s.cfg.dockerNetwork)
for id, endpoint := range network.Containers {
s.T().Logf("removing stale container %s attached to network %s", endpoint.Name, s.cfg.dockerNetwork)
if err := s.dkrPool.Client.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true, RemoveVolumes: true}); err != nil {
s.T().Logf("failed to remove stale container %s: %v", endpoint.Name, err)
}
}
s.T().Logf("removing stale network: %s", s.cfg.dockerNetwork)
if err := s.dkrPool.Client.RemoveNetwork(network.ID); err != nil && !errors.As(err, &missing) {
s.Require().NoErrorf(err, "failed to remove the stale docker network %s", s.cfg.dockerNetwork)
}
}

// removeContainerByName force-removes the container of that name, if one
Expand Down Expand Up @@ -256,6 +281,23 @@ func (s *baseTestSuite) genesisSection(appState map[string]any, path ...string)
return section
}

// setGovVotingPeriod sets the provider's governance voting period to the
// suite's and brings the gov periods that atomone-sdk validates against it
// along: the quorum timeout stays below it, the voting period extension
// covers the gap, and the two deposit throttlers update no less often than
// it. Quorum checks are off (the default check count is zero) and the
// throttlers only decay toward their floors on the clock, so the test
// proposals see none of this.
func (s *baseTestSuite) setGovVotingPeriod(appState map[string]any) {
period := fmt.Sprintf("%ds", int64(s.cfg.govVotingPeriod.Seconds()))
params := s.genesisSection(appState, "gov", "params")
params["voting_period"] = period
params["quorum_timeout"] = "0s"
params["max_voting_period_extension"] = period
s.genesisSection(appState, "gov", "params", "min_deposit_throttler")["update_period"] = period
s.genesisSection(appState, "gov", "params", "min_initial_deposit_throttler")["update_period"] = period
}

// validatorCount is the number of validator nodes each chain runs.
func (s *baseTestSuite) validatorCount() int {
if s.cfg.validators < 1 {
Expand Down
47 changes: 40 additions & 7 deletions tests/e2e/e2e_forge_equivocation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
Expand Down Expand Up @@ -34,15 +35,47 @@ func (s *ByzantineConsumerIntegrationTestSuite) consensusPrivKey(i int) cmted255
// pendingEquivocationsOf counts the consumer's pending equivocation
// punishments naming the validator at consAddr.
func (s *ByzantineConsumerIntegrationTestSuite) pendingEquivocationsOf(consumerID, consAddr string) int { //nolint:unparam // the suite drives one consumer; the helper stays generic
n, err := s.tryPendingEquivocationsOf(consumerID, consAddr)
s.Require().NoErrorf(err, "failed to count the pending equivocation punishments naming %s", consAddr)
return n
}

// tryPendingEquivocationsOf is pendingEquivocationsOf returning an error
// instead of failing the test, for the conditions of Eventually and Never,
// which run in a goroutine of their own: a failure there stalls the wait to
// its timeout.
func (s *ByzantineConsumerIntegrationTestSuite) tryPendingEquivocationsOf(consumerID, consAddr string) (int, error) {
pending, err := s.pendingEquivocationsNaming(consumerID, consAddr)
return len(pending), err
}

// pendingEquivocationsNaming lists the consumer's pending equivocation
// punishments naming the validator at consAddr.
func (s *ByzantineConsumerIntegrationTestSuite) pendingEquivocationsNaming(consumerID, consAddr string) ([]pendingEquivocationJSON, error) {
addr, err := sdk.ConsAddressFromBech32(consAddr)
s.Require().NoErrorf(err, "failed to decode %s", consAddr)
n := 0
for _, p := range s.mustPendingEquivocations(consumerID) {
if err != nil {
return nil, fmt.Errorf("decoding %s: %w", consAddr, err)
}
pending, err := s.queryPendingEquivocations(consumerID)
if err != nil {
return nil, err
}
var naming []pendingEquivocationJSON
for _, p := range pending {
if bytes.Equal(p.ProviderConsAddr, addr) {
n++
naming = append(naming, p)
}
}
return n
return naming, nil
}

// pendingEquivocationMaturity returns when the consumer's one pending
// equivocation punishment naming the validator at consAddr executes.
func (s *ByzantineConsumerIntegrationTestSuite) pendingEquivocationMaturity(consumerID, consAddr string) time.Time { //nolint:unparam // the suite drives one consumer; the helper stays generic
pending, err := s.pendingEquivocationsNaming(consumerID, consAddr)
s.Require().NoErrorf(err, "failed to read the pending equivocation punishments naming %s", consAddr)
s.Require().Lenf(pending, 1, "one pending equivocation punishment naming %s expected, got %d", consAddr, len(pending))
return pending[0].ExecutesAt
}

// testFabricatedDoubleSignJailsAndDefers: the test signs two conflicting
Expand All @@ -63,8 +96,8 @@ func (s *ByzantineConsumerIntegrationTestSuite) testFabricatedDoubleSignJailsAnd
s.submitDoubleVoteEvidence(consumerID, evidencePath, headerPath)

s.Require().Eventuallyf(func() bool {
jailed, _ := s.stakingValidatorState(val2.valoper)
return jailed
jailed, _, err := s.tryStakingValidatorState(val2.valoper)
return err == nil && jailed
}, time.Minute, 2*time.Second, "val2 was not jailed for the fabricated double-sign")
s.Require().Equalf(1, s.pendingEquivocationsOf(consumerID, val2.consAddr),
"the fabricated double-sign must queue one punishment against val2")
Expand Down
112 changes: 99 additions & 13 deletions tests/e2e/e2e_forge_fork_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,17 +152,53 @@ func (s *ByzantineConsumerIntegrationTestSuite) fabricateFork(clientID string) s

// clientStatus reads the status of one of the provider's IBC clients.
func (s *ByzantineConsumerIntegrationTestSuite) clientStatus(clientID string) string {
stdout, _, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{
status, err := s.tryClientStatus(clientID)
s.Require().NoErrorf(err, "failed to read the status of client %s", clientID)
return status
}

// tryClientStatus is clientStatus returning an error instead of failing the
// test, for the conditions of Eventually and Never, which run in a goroutine
// of their own: a failure there stalls the wait to its timeout.
func (s *ByzantineConsumerIntegrationTestSuite) tryClientStatus(clientID string) (string, error) {
stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{
providerBinary, "query", "ibc", "client", "status", clientID,
"--home", providerHomePath,
"--output", "json",
})
s.Require().NoErrorf(err, "failed to query the status of client %s", clientID)
if err != nil {
return "", fmt.Errorf("querying the status of client %s: %w (stderr=%s)", clientID, err, stderr.String())
}
var res struct {
Status string `json:"status"`
}
s.Require().NoErrorf(json.Unmarshal(stdout.Bytes(), &res), "failed to decode the client status: %s", stdout.String())
return res.Status
if err := json.Unmarshal(stdout.Bytes(), &res); err != nil {
return "", fmt.Errorf("decoding the status of client %s: %w (raw=%s)", clientID, err, stdout.String())
}
return res.Status, nil
}

// committedTxHeight returns the height of the provider block that committed
// the transaction a `-o json` broadcast response names, once
// requireTxCommitted has seen it committed.
func (s *ByzantineConsumerIntegrationTestSuite) committedTxHeight(broadcastOut []byte) int64 {
var bres struct {
TxHash string `json:"txhash"`
}
s.Require().NoErrorf(json.Unmarshal(broadcastOut, &bres), "decode broadcast response: %s", string(broadcastOut))
stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{
providerBinary, "query", "tx", bres.TxHash,
"--home", providerHomePath,
"--output", "json",
})
s.Require().NoErrorf(err, "failed to query tx %s: %s", bres.TxHash, stderr.String())
var res struct {
Height string `json:"height"`
}
s.Require().NoErrorf(json.Unmarshal(stdout.Bytes(), &res), "failed to decode tx %s: %s", bres.TxHash, stdout.String())
height, err := strconv.ParseInt(res.Height, 10, 64)
s.Require().NoErrorf(err, "tx %s height: %q", bres.TxHash, res.Height)
return height
}

// testForkFreezesAndPausesWithoutPunishing: the test signs a fork of the
Expand Down Expand Up @@ -192,6 +228,11 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkFreezesAndPausesWithoutP
}
equivocationsBefore := len(s.mustPendingEquivocations(consumerID))
downtimeBefore := len(s.queryPendingDowntimeSlashes(consumerID))
// val2's deferred punishment, queued by the fabricated double-sign,
// keeps its maturity through the pause; the fork drill requires the
// resume to move it by exactly the pause.
val2 := s.validator(2)
s.val2MaturityBeforeFork = s.pendingEquivocationMaturity(consumerID, val2.consAddr)

path := s.fabricateFork(clientID)
stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{
Expand All @@ -207,9 +248,14 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkFreezesAndPausesWithoutP
})
s.Require().NoErrorf(err, "failed to submit the fabricated misbehaviour: stderr=%s", stderr.String())
s.requireTxCommitted(stdout.Bytes())
// The message freezes the client and pauses the consumer in its own
// block, whose time is the pause's.
s.forkPausedAt = s.providerBlockTime(s.committedTxHeight(stdout.Bytes()))

s.Require().Equalf("Frozen", s.clientStatus(clientID), "the fork must freeze client %s", clientID)
s.Require().Equalf(phasePaused, s.queryProviderConsumerPhase(consumerID), "the fork must pause the consumer")
s.Require().Truef(s.pendingEquivocationMaturity(consumerID, val2.consAddr).Equal(s.val2MaturityBeforeFork),
"the pause must leave val2's deferred punishment at its maturity")
for i, was := range before {
v := s.validator(i)
jailed, tokens := s.stakingValidatorState(v.valoper)
Expand Down Expand Up @@ -244,15 +290,18 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkFreezesAndPausesWithoutP
// the client stays frozen. Once the trusting period has elapsed since the
// pause the same proposal passes: the client is active and the relayer
// keeps updating it with the consumer's real headers, the consumer is
// launched and its acks flow again, and nobody was punished. The forge
// suite's consumer carries a short unbonding period so this wait fits the
// job (create_consumer_forge.json).
// launched and its acks flow again, nobody was punished, and val2's deferred
// punishment executes as long after the resume as it would have after the
// pause. The forge suite's consumer carries a short unbonding period so this
// wait fits the job (create_consumer_forge.json).
func (s *ByzantineConsumerIntegrationTestSuite) testForkRecoveryWaitsOutTheTrustingPeriod() {
s.Run("a recovery and resume is refused inside the trusting period, and passes once it has elapsed", func() {
const consumerID = "0"
subject := s.queryConsumerClientID(consumerID)
s.Require().Equalf("Frozen", s.clientStatus(subject), "the drill starts from the fork's freeze")
s.Require().Equalf(phasePaused, s.queryProviderConsumerPhase(consumerID), "the drill starts from the fork's pause")
s.Require().Falsef(s.forkPausedAt.IsZero(), "the drill starts from the pause the fork scenario recorded")
val2 := s.validator(2)

type state struct {
jailed bool
Expand All @@ -275,6 +324,8 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkRecoveryWaitsOutTheTrust
s.Require().Containsf(failure, "trusting period", "the resume must be refused inside the trusting period: %s", failure)
s.Require().Equalf("Frozen", s.clientStatus(subject), "a failed proposal rolls the recovery back with it")
s.Require().Equalf(phasePaused, s.queryProviderConsumerPhase(consumerID), "a refused resume leaves the consumer paused")
s.Require().Truef(s.pendingEquivocationMaturity(consumerID, val2.consAddr).Equal(s.val2MaturityBeforeFork),
"a refused resume leaves val2's deferred punishment at its maturity")

resumableAt := resumableAtFromFailure(failure)
s.Require().Falsef(resumableAt.IsZero(), "the refusal must say when the resume is allowed: %s", failure)
Expand All @@ -292,15 +343,35 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkRecoveryWaitsOutTheTrust
substitute = s.newProviderClientOfTheConsumer()
s.T().Logf("recovering the frozen client %s onto the fresh substitute %s", subject, substitute)
proposal = s.recoverAndResumeProposalJSON(consumerID, subject, substitute)
resumeSubmittedAt, err := s.queryProviderBlockHeight()
s.Require().NoError(err, "failed to read the provider height")
s.submitProposalWithVote(proposal, "yes", "PROPOSAL_STATUS_PASSED")
s.Require().Equalf("Active", s.clientStatus(subject), "the recovered client must be active")
s.Require().Equalf(phaseLaunched, s.queryProviderConsumerPhase(consumerID), "the consumer must be launched again")

// The resume thaws what the pause froze: val2's deferred punishment
// executes as long after the resume as it would have after the
// pause, the pause being the time from the block the fork paused the
// consumer in to the block the proposal resumed it in.
latest, err := s.queryProviderBlockHeight()
s.Require().NoError(err, "failed to read the provider height")
resumed := s.providerBlockEventHeights(resumeSubmittedAt, latest, resumedEventType, map[string]string{"consumer_id": consumerID})
s.Require().Lenf(resumed, 1, "one resume of consumer %s expected in provider blocks %d..%d, got %v", consumerID, resumeSubmittedAt, latest, resumed)
pausedFor := s.providerBlockTime(resumed[0]).Sub(s.forkPausedAt)
wantMaturity := s.val2MaturityBeforeFork.Add(pausedFor)
maturity := s.pendingEquivocationMaturity(consumerID, val2.consAddr)
s.Require().Truef(maturity.Equal(wantMaturity),
"val2's deferred punishment must execute at %s, its maturity before the fork moved by the pause of %s, not at %s",
wantMaturity.Format(time.RFC3339Nano), pausedFor, maturity.Format(time.RFC3339Nano))
s.T().Logf("the resume moved val2's deferred punishment by the pause of %s", pausedFor.Round(time.Second))

// The recovered client verifies the consumer's real headers: the
// relayer keeps moving it.
recoveredAt := s.clientLatestHeight(subject)
s.Require().Eventuallyf(func() bool { return s.clientLatestHeight(subject) > recoveredAt },
3*time.Minute, 3*time.Second, "the recovered client %s was never updated past height %d", subject, recoveredAt)
s.Require().Eventuallyf(func() bool {
height, err := s.tryClientLatestHeight(subject)
return err == nil && height > recoveredAt
}, 3*time.Minute, 3*time.Second, "the recovered client %s was never updated past height %d", subject, recoveredAt)
// And the consumer is served again: its acks flow.
s.waitForConsumerSync(consumerID)

Expand Down Expand Up @@ -379,19 +450,34 @@ func resumableAtFromFailure(failure string) time.Time {
// clientLatestHeight returns the latest height of a provider-side IBC
// client's state.
func (s *ByzantineConsumerIntegrationTestSuite) clientLatestHeight(clientID string) int64 {
height, err := s.tryClientLatestHeight(clientID)
s.Require().NoErrorf(err, "failed to read the latest height of client %s", clientID)
return height
}

// tryClientLatestHeight is clientLatestHeight returning an error instead of
// failing the test, for the conditions of Eventually and Never, which run in
// a goroutine of their own: a failure there stalls the wait to its timeout.
func (s *ByzantineConsumerIntegrationTestSuite) tryClientLatestHeight(clientID string) (int64, error) {
stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{
providerBinary, "query", "ibc", "client", "state", clientID, "--home", providerHomePath, "-o", "json",
})
s.Require().NoErrorf(err, "failed to query client %s: %s", clientID, stderr.String())
if err != nil {
return 0, fmt.Errorf("querying client %s: %w (stderr=%s)", clientID, err, stderr.String())
}
var res struct {
ClientState struct {
LatestHeight struct {
RevisionHeight string `json:"revision_height"`
} `json:"latest_height"`
} `json:"client_state"`
}
s.Require().NoErrorf(json.Unmarshal(stdout.Bytes(), &res), "failed to decode client %s: %s", clientID, stdout.String())
if err := json.Unmarshal(stdout.Bytes(), &res); err != nil {
return 0, fmt.Errorf("decoding client %s: %w (raw=%s)", clientID, err, stdout.String())
}
height, err := strconv.ParseInt(res.ClientState.LatestHeight.RevisionHeight, 10, 64)
s.Require().NoErrorf(err, "client %s latest height: %q", clientID, res.ClientState.LatestHeight.RevisionHeight)
return height
if err != nil {
return 0, fmt.Errorf("client %s latest height: %q: %w", clientID, res.ClientState.LatestHeight.RevisionHeight, err)
}
return height, nil
}
Loading
Loading