diff --git a/.gitignore b/.gitignore index b428f07e..80f84a31 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ Icon? coverage.out build/ hermes +tests/e2e/forge/forge # IDEs and editors .vscode/ diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 3a434126..bd693a98 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -130,8 +130,9 @@ misbehaviour are accepted only from a launched consumer): a resume is refused while the client is frozen; 4. governance recovers the frozen client onto a substitute and resumes the consumer in one proposal: refused inside the client's trusting period and - rolled back, passed once it has elapsed; the consumer is served again and - nobody is punished; + rolled back, passed once it has elapsed; the consumer is served again, + nobody is punished, and the deferred punishment's clock resumes where the + pause stopped it; 5. the consumer restarts under the next revision of its chain id; the provider's client expires and governance recovers it onto a client of the renamed chain; the next provider block pauses the consumer for the chain diff --git a/tests/e2e/base_suite_test.go b/tests/e2e/base_suite_test.go index ce0a8877..b76b3d12 100644 --- a/tests/e2e/base_suite_test.go +++ b/tests/e2e/base_suite_test.go @@ -174,8 +174,33 @@ func (s *baseTestSuite) cleanupStaleContainers() { for _, name := range staleNames { s.removeContainerByName(name) } - // Also remove stale network - _ = s.dkrPool.Client.RemoveNetwork(s.cfg.dockerNetwork) + s.removeStaleNetwork() +} + +// removeStaleNetwork removes the suite's docker network if a previous run +// left it behind, with whatever containers are still attached to it: a +// scenario's ephemeral export or validate container outlives a run killed +// while it ran, docker refuses to remove a network with an endpoint, and +// the name stays taken, so bringUpSuite would fail to create the network. +// A network that cannot be removed fails the suite here, with docker's +// reason, instead of at that creation. +func (s *baseTestSuite) removeStaleNetwork() { + var missing *docker.NoSuchNetwork + network, err := s.dkrPool.Client.NetworkInfo(s.cfg.dockerNetwork) + if errors.As(err, &missing) { + return + } + s.Require().NoErrorf(err, "failed to inspect the docker network %s", s.cfg.dockerNetwork) + for id, endpoint := range network.Containers { + s.T().Logf("removing stale container %s attached to network %s", endpoint.Name, s.cfg.dockerNetwork) + if err := s.dkrPool.Client.RemoveContainer(docker.RemoveContainerOptions{ID: id, Force: true, RemoveVolumes: true}); err != nil { + s.T().Logf("failed to remove stale container %s: %v", endpoint.Name, err) + } + } + s.T().Logf("removing stale network: %s", s.cfg.dockerNetwork) + if err := s.dkrPool.Client.RemoveNetwork(network.ID); err != nil && !errors.As(err, &missing) { + s.Require().NoErrorf(err, "failed to remove the stale docker network %s", s.cfg.dockerNetwork) + } } // removeContainerByName force-removes the container of that name, if one @@ -256,6 +281,23 @@ func (s *baseTestSuite) genesisSection(appState map[string]any, path ...string) return section } +// setGovVotingPeriod sets the provider's governance voting period to the +// suite's and brings the gov periods that atomone-sdk validates against it +// along: the quorum timeout stays below it, the voting period extension +// covers the gap, and the two deposit throttlers update no less often than +// it. Quorum checks are off (the default check count is zero) and the +// throttlers only decay toward their floors on the clock, so the test +// proposals see none of this. +func (s *baseTestSuite) setGovVotingPeriod(appState map[string]any) { + period := fmt.Sprintf("%ds", int64(s.cfg.govVotingPeriod.Seconds())) + params := s.genesisSection(appState, "gov", "params") + params["voting_period"] = period + params["quorum_timeout"] = "0s" + params["max_voting_period_extension"] = period + s.genesisSection(appState, "gov", "params", "min_deposit_throttler")["update_period"] = period + s.genesisSection(appState, "gov", "params", "min_initial_deposit_throttler")["update_period"] = period +} + // validatorCount is the number of validator nodes each chain runs. func (s *baseTestSuite) validatorCount() int { if s.cfg.validators < 1 { diff --git a/tests/e2e/e2e_forge_equivocation_test.go b/tests/e2e/e2e_forge_equivocation_test.go index b4f44983..6240a75c 100644 --- a/tests/e2e/e2e_forge_equivocation_test.go +++ b/tests/e2e/e2e_forge_equivocation_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/base64" "encoding/json" + "fmt" "os" "path/filepath" "time" @@ -34,15 +35,47 @@ func (s *ByzantineConsumerIntegrationTestSuite) consensusPrivKey(i int) cmted255 // pendingEquivocationsOf counts the consumer's pending equivocation // punishments naming the validator at consAddr. func (s *ByzantineConsumerIntegrationTestSuite) pendingEquivocationsOf(consumerID, consAddr string) int { //nolint:unparam // the suite drives one consumer; the helper stays generic + n, err := s.tryPendingEquivocationsOf(consumerID, consAddr) + s.Require().NoErrorf(err, "failed to count the pending equivocation punishments naming %s", consAddr) + return n +} + +// tryPendingEquivocationsOf is pendingEquivocationsOf returning an error +// instead of failing the test, for the conditions of Eventually and Never, +// which run in a goroutine of their own: a failure there stalls the wait to +// its timeout. +func (s *ByzantineConsumerIntegrationTestSuite) tryPendingEquivocationsOf(consumerID, consAddr string) (int, error) { + pending, err := s.pendingEquivocationsNaming(consumerID, consAddr) + return len(pending), err +} + +// pendingEquivocationsNaming lists the consumer's pending equivocation +// punishments naming the validator at consAddr. +func (s *ByzantineConsumerIntegrationTestSuite) pendingEquivocationsNaming(consumerID, consAddr string) ([]pendingEquivocationJSON, error) { addr, err := sdk.ConsAddressFromBech32(consAddr) - s.Require().NoErrorf(err, "failed to decode %s", consAddr) - n := 0 - for _, p := range s.mustPendingEquivocations(consumerID) { + if err != nil { + return nil, fmt.Errorf("decoding %s: %w", consAddr, err) + } + pending, err := s.queryPendingEquivocations(consumerID) + if err != nil { + return nil, err + } + var naming []pendingEquivocationJSON + for _, p := range pending { if bytes.Equal(p.ProviderConsAddr, addr) { - n++ + naming = append(naming, p) } } - return n + return naming, nil +} + +// pendingEquivocationMaturity returns when the consumer's one pending +// equivocation punishment naming the validator at consAddr executes. +func (s *ByzantineConsumerIntegrationTestSuite) pendingEquivocationMaturity(consumerID, consAddr string) time.Time { //nolint:unparam // the suite drives one consumer; the helper stays generic + pending, err := s.pendingEquivocationsNaming(consumerID, consAddr) + s.Require().NoErrorf(err, "failed to read the pending equivocation punishments naming %s", consAddr) + s.Require().Lenf(pending, 1, "one pending equivocation punishment naming %s expected, got %d", consAddr, len(pending)) + return pending[0].ExecutesAt } // testFabricatedDoubleSignJailsAndDefers: the test signs two conflicting @@ -63,8 +96,8 @@ func (s *ByzantineConsumerIntegrationTestSuite) testFabricatedDoubleSignJailsAnd s.submitDoubleVoteEvidence(consumerID, evidencePath, headerPath) s.Require().Eventuallyf(func() bool { - jailed, _ := s.stakingValidatorState(val2.valoper) - return jailed + jailed, _, err := s.tryStakingValidatorState(val2.valoper) + return err == nil && jailed }, time.Minute, 2*time.Second, "val2 was not jailed for the fabricated double-sign") s.Require().Equalf(1, s.pendingEquivocationsOf(consumerID, val2.consAddr), "the fabricated double-sign must queue one punishment against val2") diff --git a/tests/e2e/e2e_forge_fork_test.go b/tests/e2e/e2e_forge_fork_test.go index d23edd99..06d01184 100644 --- a/tests/e2e/e2e_forge_fork_test.go +++ b/tests/e2e/e2e_forge_fork_test.go @@ -152,17 +152,53 @@ func (s *ByzantineConsumerIntegrationTestSuite) fabricateFork(clientID string) s // clientStatus reads the status of one of the provider's IBC clients. func (s *ByzantineConsumerIntegrationTestSuite) clientStatus(clientID string) string { - stdout, _, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ + status, err := s.tryClientStatus(clientID) + s.Require().NoErrorf(err, "failed to read the status of client %s", clientID) + return status +} + +// tryClientStatus is clientStatus returning an error instead of failing the +// test, for the conditions of Eventually and Never, which run in a goroutine +// of their own: a failure there stalls the wait to its timeout. +func (s *ByzantineConsumerIntegrationTestSuite) tryClientStatus(clientID string) (string, error) { + stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ providerBinary, "query", "ibc", "client", "status", clientID, "--home", providerHomePath, "--output", "json", }) - s.Require().NoErrorf(err, "failed to query the status of client %s", clientID) + if err != nil { + return "", fmt.Errorf("querying the status of client %s: %w (stderr=%s)", clientID, err, stderr.String()) + } var res struct { Status string `json:"status"` } - s.Require().NoErrorf(json.Unmarshal(stdout.Bytes(), &res), "failed to decode the client status: %s", stdout.String()) - return res.Status + if err := json.Unmarshal(stdout.Bytes(), &res); err != nil { + return "", fmt.Errorf("decoding the status of client %s: %w (raw=%s)", clientID, err, stdout.String()) + } + return res.Status, nil +} + +// committedTxHeight returns the height of the provider block that committed +// the transaction a `-o json` broadcast response names, once +// requireTxCommitted has seen it committed. +func (s *ByzantineConsumerIntegrationTestSuite) committedTxHeight(broadcastOut []byte) int64 { + var bres struct { + TxHash string `json:"txhash"` + } + s.Require().NoErrorf(json.Unmarshal(broadcastOut, &bres), "decode broadcast response: %s", string(broadcastOut)) + stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ + providerBinary, "query", "tx", bres.TxHash, + "--home", providerHomePath, + "--output", "json", + }) + s.Require().NoErrorf(err, "failed to query tx %s: %s", bres.TxHash, stderr.String()) + var res struct { + Height string `json:"height"` + } + s.Require().NoErrorf(json.Unmarshal(stdout.Bytes(), &res), "failed to decode tx %s: %s", bres.TxHash, stdout.String()) + height, err := strconv.ParseInt(res.Height, 10, 64) + s.Require().NoErrorf(err, "tx %s height: %q", bres.TxHash, res.Height) + return height } // testForkFreezesAndPausesWithoutPunishing: the test signs a fork of the @@ -192,6 +228,11 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkFreezesAndPausesWithoutP } equivocationsBefore := len(s.mustPendingEquivocations(consumerID)) downtimeBefore := len(s.queryPendingDowntimeSlashes(consumerID)) + // val2's deferred punishment, queued by the fabricated double-sign, + // keeps its maturity through the pause; the fork drill requires the + // resume to move it by exactly the pause. + val2 := s.validator(2) + s.val2MaturityBeforeFork = s.pendingEquivocationMaturity(consumerID, val2.consAddr) path := s.fabricateFork(clientID) stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ @@ -207,9 +248,14 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkFreezesAndPausesWithoutP }) s.Require().NoErrorf(err, "failed to submit the fabricated misbehaviour: stderr=%s", stderr.String()) s.requireTxCommitted(stdout.Bytes()) + // The message freezes the client and pauses the consumer in its own + // block, whose time is the pause's. + s.forkPausedAt = s.providerBlockTime(s.committedTxHeight(stdout.Bytes())) s.Require().Equalf("Frozen", s.clientStatus(clientID), "the fork must freeze client %s", clientID) s.Require().Equalf(phasePaused, s.queryProviderConsumerPhase(consumerID), "the fork must pause the consumer") + s.Require().Truef(s.pendingEquivocationMaturity(consumerID, val2.consAddr).Equal(s.val2MaturityBeforeFork), + "the pause must leave val2's deferred punishment at its maturity") for i, was := range before { v := s.validator(i) jailed, tokens := s.stakingValidatorState(v.valoper) @@ -244,15 +290,18 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkFreezesAndPausesWithoutP // the client stays frozen. Once the trusting period has elapsed since the // pause the same proposal passes: the client is active and the relayer // keeps updating it with the consumer's real headers, the consumer is -// launched and its acks flow again, and nobody was punished. The forge -// suite's consumer carries a short unbonding period so this wait fits the -// job (create_consumer_forge.json). +// launched and its acks flow again, nobody was punished, and val2's deferred +// punishment executes as long after the resume as it would have after the +// pause. The forge suite's consumer carries a short unbonding period so this +// wait fits the job (create_consumer_forge.json). func (s *ByzantineConsumerIntegrationTestSuite) testForkRecoveryWaitsOutTheTrustingPeriod() { s.Run("a recovery and resume is refused inside the trusting period, and passes once it has elapsed", func() { const consumerID = "0" subject := s.queryConsumerClientID(consumerID) s.Require().Equalf("Frozen", s.clientStatus(subject), "the drill starts from the fork's freeze") s.Require().Equalf(phasePaused, s.queryProviderConsumerPhase(consumerID), "the drill starts from the fork's pause") + s.Require().Falsef(s.forkPausedAt.IsZero(), "the drill starts from the pause the fork scenario recorded") + val2 := s.validator(2) type state struct { jailed bool @@ -275,6 +324,8 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkRecoveryWaitsOutTheTrust s.Require().Containsf(failure, "trusting period", "the resume must be refused inside the trusting period: %s", failure) s.Require().Equalf("Frozen", s.clientStatus(subject), "a failed proposal rolls the recovery back with it") s.Require().Equalf(phasePaused, s.queryProviderConsumerPhase(consumerID), "a refused resume leaves the consumer paused") + s.Require().Truef(s.pendingEquivocationMaturity(consumerID, val2.consAddr).Equal(s.val2MaturityBeforeFork), + "a refused resume leaves val2's deferred punishment at its maturity") resumableAt := resumableAtFromFailure(failure) s.Require().Falsef(resumableAt.IsZero(), "the refusal must say when the resume is allowed: %s", failure) @@ -292,15 +343,35 @@ func (s *ByzantineConsumerIntegrationTestSuite) testForkRecoveryWaitsOutTheTrust substitute = s.newProviderClientOfTheConsumer() s.T().Logf("recovering the frozen client %s onto the fresh substitute %s", subject, substitute) proposal = s.recoverAndResumeProposalJSON(consumerID, subject, substitute) + resumeSubmittedAt, err := s.queryProviderBlockHeight() + s.Require().NoError(err, "failed to read the provider height") s.submitProposalWithVote(proposal, "yes", "PROPOSAL_STATUS_PASSED") s.Require().Equalf("Active", s.clientStatus(subject), "the recovered client must be active") s.Require().Equalf(phaseLaunched, s.queryProviderConsumerPhase(consumerID), "the consumer must be launched again") + // The resume thaws what the pause froze: val2's deferred punishment + // executes as long after the resume as it would have after the + // pause, the pause being the time from the block the fork paused the + // consumer in to the block the proposal resumed it in. + latest, err := s.queryProviderBlockHeight() + s.Require().NoError(err, "failed to read the provider height") + resumed := s.providerBlockEventHeights(resumeSubmittedAt, latest, resumedEventType, map[string]string{"consumer_id": consumerID}) + s.Require().Lenf(resumed, 1, "one resume of consumer %s expected in provider blocks %d..%d, got %v", consumerID, resumeSubmittedAt, latest, resumed) + pausedFor := s.providerBlockTime(resumed[0]).Sub(s.forkPausedAt) + wantMaturity := s.val2MaturityBeforeFork.Add(pausedFor) + maturity := s.pendingEquivocationMaturity(consumerID, val2.consAddr) + s.Require().Truef(maturity.Equal(wantMaturity), + "val2's deferred punishment must execute at %s, its maturity before the fork moved by the pause of %s, not at %s", + wantMaturity.Format(time.RFC3339Nano), pausedFor, maturity.Format(time.RFC3339Nano)) + s.T().Logf("the resume moved val2's deferred punishment by the pause of %s", pausedFor.Round(time.Second)) + // The recovered client verifies the consumer's real headers: the // relayer keeps moving it. recoveredAt := s.clientLatestHeight(subject) - s.Require().Eventuallyf(func() bool { return s.clientLatestHeight(subject) > recoveredAt }, - 3*time.Minute, 3*time.Second, "the recovered client %s was never updated past height %d", subject, recoveredAt) + s.Require().Eventuallyf(func() bool { + height, err := s.tryClientLatestHeight(subject) + return err == nil && height > recoveredAt + }, 3*time.Minute, 3*time.Second, "the recovered client %s was never updated past height %d", subject, recoveredAt) // And the consumer is served again: its acks flow. s.waitForConsumerSync(consumerID) @@ -379,10 +450,21 @@ func resumableAtFromFailure(failure string) time.Time { // clientLatestHeight returns the latest height of a provider-side IBC // client's state. func (s *ByzantineConsumerIntegrationTestSuite) clientLatestHeight(clientID string) int64 { + height, err := s.tryClientLatestHeight(clientID) + s.Require().NoErrorf(err, "failed to read the latest height of client %s", clientID) + return height +} + +// tryClientLatestHeight is clientLatestHeight returning an error instead of +// failing the test, for the conditions of Eventually and Never, which run in +// a goroutine of their own: a failure there stalls the wait to its timeout. +func (s *ByzantineConsumerIntegrationTestSuite) tryClientLatestHeight(clientID string) (int64, error) { stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ providerBinary, "query", "ibc", "client", "state", clientID, "--home", providerHomePath, "-o", "json", }) - s.Require().NoErrorf(err, "failed to query client %s: %s", clientID, stderr.String()) + if err != nil { + return 0, fmt.Errorf("querying client %s: %w (stderr=%s)", clientID, err, stderr.String()) + } var res struct { ClientState struct { LatestHeight struct { @@ -390,8 +472,12 @@ func (s *ByzantineConsumerIntegrationTestSuite) clientLatestHeight(clientID stri } `json:"latest_height"` } `json:"client_state"` } - s.Require().NoErrorf(json.Unmarshal(stdout.Bytes(), &res), "failed to decode client %s: %s", clientID, stdout.String()) + if err := json.Unmarshal(stdout.Bytes(), &res); err != nil { + return 0, fmt.Errorf("decoding client %s: %w (raw=%s)", clientID, err, stdout.String()) + } height, err := strconv.ParseInt(res.ClientState.LatestHeight.RevisionHeight, 10, 64) - s.Require().NoErrorf(err, "client %s latest height: %q", clientID, res.ClientState.LatestHeight.RevisionHeight) - return height + if err != nil { + return 0, fmt.Errorf("client %s latest height: %q: %w", clientID, res.ClientState.LatestHeight.RevisionHeight, err) + } + return height, nil } diff --git a/tests/e2e/e2e_forge_revision_bump_test.go b/tests/e2e/e2e_forge_revision_bump_test.go index 82025b13..ebbde12f 100644 --- a/tests/e2e/e2e_forge_revision_bump_test.go +++ b/tests/e2e/e2e_forge_revision_bump_test.go @@ -61,8 +61,10 @@ func (s *ByzantineConsumerIntegrationTestSuite) testRevisionBumpIsServedOnlyAfte // unupdated and expires with its trusting period (two thirds of the // consumer's unbonding period, create_consumer_forge.json). s.T().Logf("waiting for the declared client %s to expire...", subject) - s.Require().Eventuallyf(func() bool { return s.clientStatus(subject) == clientExpired }, - 12*time.Minute, 10*time.Second, "the declared client %s never expired", subject) + s.Require().Eventuallyf(func() bool { + status, err := s.tryClientStatus(subject) + return err == nil && status == clientExpired + }, 12*time.Minute, 10*time.Second, "the declared client %s never expired", subject) s.Require().Equalf(phaseLaunched, s.queryProviderConsumerPhase(consumerID), "an expired client changes no phase") // The relayer is set up again for the renamed chain, as an operator @@ -100,12 +102,21 @@ func (s *ByzantineConsumerIntegrationTestSuite) testRevisionBumpIsServedOnlyAfte "consumer_chain_id": oldChainID, }) s.T().Logf("the consumer was paused for the chain id mismatch at provider height %d", pausedAt) - s.Require().Emptyf(s.queryPendingDowntimeSlashes(consumerID), "the mismatch pause discards the consumer's downtime accusations") - - // The paused consumer is served nothing, and neither the liveness - // sweep nor the mismatch check acts on it again meanwhile. The pause - // also schedules the consumer's auto-stop max_pause_duration out, - // which this suite leaves at its default of 45 days. + // No accusation is pending against the paused consumer. None was + // before the recovery either (four live validators, the lie long + // over, val1's accusation challenged away), so this shows the pause + // adding none, not discarding any; the discard is pinned by + // TestPauseConsumersWithMismatchedClients in the provider keeper's + // tests. + s.Require().Emptyf(s.queryPendingDowntimeSlashes(consumerID), "no downtime accusation may be pending against the paused consumer") + + // The paused consumer is served nothing, and the mismatch check does + // not act on it again meanwhile: the pause is reported once. The + // liveness sweep is out of this window's reach whatever it did, the + // suite's grace being two thirds of the provider's 21-day unbonding + // period, so the watch says nothing about it. The pause also + // schedules the consumer's auto-stop max_pause_duration out, which + // this suite leaves at its default of 45 days. sentBefore, appliedBefore := s.requireConsumerNotServed(subject, consumerID, pausedAt) // 4. One governance proposal renames the consumer to the chain id its @@ -134,8 +145,10 @@ func (s *ByzantineConsumerIntegrationTestSuite) testRevisionBumpIsServedOnlyAfte s.Require().Positive(boundaryHeight, "the boundary is the client's height at the rename") s.Require().LessOrEqual(boundaryHeight, s.clientLatestHeight(subject), "the boundary cannot be past the client") - s.Require().Eventuallyf(func() bool { return s.nextSequenceSend(subject) > sentBefore }, - 2*time.Minute, 3*time.Second, "the provider sent no VSC packet over client %s after the rename", subject) + s.Require().Eventuallyf(func() bool { + sent, err := s.tryNextSequenceSend(subject) + return err == nil && sent > sentBefore + }, 2*time.Minute, 3*time.Second, "the provider sent no VSC packet over client %s after the rename", subject) s.Require().Eventuallyf(func() bool { return s.highestAppliedVscID() > appliedBefore }, 3*time.Minute, 3*time.Second, "the renamed consumer applied no VSC packet after the rename") s.waitForConsumerSync(consumerID) @@ -159,7 +172,8 @@ func (s *ByzantineConsumerIntegrationTestSuite) testRevisionBumpIsServedOnlyAfte evidencePath, headerPath := s.fabricateDoubleVote(s.consensusPrivKey(2), oldChainID, haltedAt-5, "forge-val2-old-chain-id") s.submitDoubleVoteEvidence(consumerID, evidencePath, headerPath) s.Require().Eventuallyf(func() bool { - return s.pendingEquivocationsOf(consumerID, val2.consAddr) == 2 + n, err := s.tryPendingEquivocationsOf(consumerID, val2.consAddr) + return err == nil && n == 2 }, time.Minute, 2*time.Second, "the double vote signed over the old chain id must queue a second punishment against val2") _, tokensAfter := s.stakingValidatorState(val2.valoper) s.Require().Equalf(tokensBefore, tokensAfter, "the punishment must be deferred, not executed") @@ -328,12 +342,11 @@ func (s *ByzantineConsumerIntegrationTestSuite) requireMismatchPause(from int64, // the consumer was paused at height pausedAt and requires that it serves the // consumer nothing: the declared client's send sequence stays put, the // consumer's applied valset update id stops advancing, the consumer stays -// paused (the liveness sweep does not stop it), and the provider neither -// pauses it again nor repeats the mismatch event. The relayer may still be -// delivering the packets the provider sent before the client expired, which -// the consumer applies late; the watch starts over while that backlog lands. -// Returns the send sequence and the applied valset update id the window -// settled at. +// paused, and the provider neither pauses it again nor repeats the mismatch +// event. The relayer may still be delivering the packets the provider sent +// before the client expired, which the consumer applies late; the watch +// starts over while that backlog lands. Returns the send sequence and the +// applied valset update id the window settled at. func (s *ByzantineConsumerIntegrationTestSuite) requireConsumerNotServed(subject, consumerID string, pausedAt int64) (sent, applied uint64) { const epochs = 6 const window = epochs * fourValidatorBlocksPerEpoch diff --git a/tests/e2e/e2e_forge_suite_test.go b/tests/e2e/e2e_forge_suite_test.go index 7d2b23d7..109b53a3 100644 --- a/tests/e2e/e2e_forge_suite_test.go +++ b/tests/e2e/e2e_forge_suite_test.go @@ -43,6 +43,13 @@ type ByzantineConsumerIntegrationTestSuite struct { // the suite's last check compares against: nothing the lying consumer // did may have cost val1 anything. val1StakeBeforeLie string + + // val2MaturityBeforeFork is when val2's deferred punishment was to + // execute before the fork, and forkPausedAt the time of the provider + // block the fork paused the consumer in: the fork drill requires the + // resume to have moved the punishment by exactly the pause. + val2MaturityBeforeFork time.Time + forkPausedAt time.Time } func TestByzantineConsumerIntegrationTestSuite(t *testing.T) { diff --git a/tests/e2e/e2e_multival_refusal_test.go b/tests/e2e/e2e_multival_refusal_test.go index e3f10abc..1c7010fc 100644 --- a/tests/e2e/e2e_multival_refusal_test.go +++ b/tests/e2e/e2e_multival_refusal_test.go @@ -3,8 +3,21 @@ package e2e import ( "strconv" "time" + + sdk "github.com/cosmos/cosmos-sdk/types" ) +// multivalRefusalSelfDelegation is the power val0 self-delegates at the start +// of the refusal scenario, on top of the thousand every validator holds from +// genesis. With val1 at 1100 after the delegation of scenario 2 and val2 at +// 990 after the slash of scenario 4, val0 then holds 3100 of 6190, half of +// the bonded power, while val2 and val3 together hold 1990, under a third: +// one validator of four refusing pauses the consumer and two do not, the +// opposite of what counting heads would give. val0's node is never stopped +// afterwards, and the outages the later scenarios take (val3's and val1's +// restarts, val3's removal) leave over four fifths of the power online. +const multivalRefusalSelfDelegation = int64(2100) + // setConsumerRefusalFrom records or withdraws validator i's refusal of the // consumer, signed by its operator key in its own provider node, and waits // for the transaction to commit. @@ -23,37 +36,99 @@ func (s *MultiValidatorIntegrationTestSuite) setConsumerRefusalFrom(i int, consu s.requireTxCommittedOn(container, providerBinary, providerHomePath, stdout.Bytes()) } +// selfDelegateFrom delegates power to validator i from its own operator +// account, signed by its operator key in its own provider node, and waits +// for the transaction to commit. +func (s *MultiValidatorIntegrationTestSuite) selfDelegateFrom(i int, valoper string, power int64) { + amount := sdk.TokensFromConsensusPower(power, sdk.DefaultPowerReduction).String() + bondDenom + container := s.providerValRes[i].Container.ID + stdout, stderr, err := s.dockerExec(container, []string{ + providerBinary, "tx", "staking", "delegate", valoper, amount, + "--from", "val", + "--home", providerHomePath, + "--keyring-backend", "test", + "--chain-id", s.cfg.providerChainID, + "--gas", "auto", + "--gas-adjustment", "1.5", + "--fees", "10000" + bondDenom, + "-y", "-o", "json", + }) + s.Require().NoErrorf(err, "val%d failed to self-delegate %s: stderr=%s", i, amount, stderr.String()) + s.requireTxCommittedOn(container, providerBinary, providerHomePath, stdout.Bytes()) +} + +// providerPowerShare returns the share of the provider's consensus power the +// validators at consAddrs hold together. +func (s *MultiValidatorIntegrationTestSuite) providerPowerShare(consAddrs ...string) float64 { + vals, err := s.queryProviderNetValidators() + s.Require().NoError(err, "failed to query the provider's validators") + powers := s.valAddrPowerMap(vals) + var held, total int64 + for _, power := range powers { + total += power + } + for _, addr := range consAddrs { + power, ok := powers[addr] + s.Require().Truef(ok, "%s is not in the provider's consensus set", addr) + held += power + } + return float64(held) / float64(total) +} + // testRefusalThresholdIsStakeWeighted: the refusal pause threshold counts -// bonded stake. One of four validators refusing holds about a quarter of -// it, under the one-third threshold, and the consumer stays launched; a -// second refusing brings the coalition to about half, and the consumer -// pauses. Both withdraw, governance resumes the consumer, and it syncs -// again. The delegation of scenario 2 and the slash of scenario 4 leave the -// two shares on the same sides of the threshold. +// bonded stake, not refusing validators. val0 self-delegates up to half of +// the bonded power (multivalRefusalSelfDelegation). val2 and val3 refuse the +// consumer, two validators of four with under a third of the power between +// them, and the consumer stays launched; they withdraw and val0 refuses +// alone, one validator of four with half of the power, and the consumer +// pauses. val0 withdraws, governance resumes the consumer, and it syncs +// again. func (s *MultiValidatorIntegrationTestSuite) testRefusalThresholdIsStakeWeighted() { - s.Run("the refusal pause threshold counts stake: one of four refusing leaves the consumer launched, two pause it", func() { + s.Run("the refusal pause threshold counts stake: two of four refusing with under a third leave the consumer launched, one with half pauses it", func() { const consumerID = "0" s.Require().Equal(phaseLaunched, s.queryProviderConsumerPhase(consumerID), "the consumer must be launched before the refusal scenario") + val0, val2, val3 := s.validator(0), s.validator(2), s.validator(3) - s.T().Log("val0 refuses the consumer: about a quarter of the bonded power...") - s.setConsumerRefusalFrom(0, consumerID, true) + s.T().Log("val0 self-delegates up to half of the bonded power...") + powerBefore, ok := s.providerPowerOf(val0.consAddr) + s.Require().Truef(ok, "val0 (%s) is not in the provider's consensus set", val0.consAddr) + s.selfDelegateFrom(0, val0.valoper, multivalRefusalSelfDelegation) + s.Require().Eventuallyf(func() bool { + power, ok := s.providerPowerOf(val0.consAddr) + return ok && power == powerBefore+multivalRefusalSelfDelegation + }, time.Minute, 2*time.Second, + "the provider's consensus set never showed val0 at %d power", powerBefore+multivalRefusalSelfDelegation) + + // The shares the two refusals are judged by, on the powers the + // earlier scenarios left: the pair's under the one-third threshold, + // val0's alone at or above it. + pair, alone := s.providerPowerShare(val2.consAddr, val3.consAddr), s.providerPowerShare(val0.consAddr) + s.T().Logf("val2 and val3 hold %.3f of the bonded power between them, val0 %.3f alone", pair, alone) + s.Require().Lessf(pair, 1.0/3, "val2 and val3 must hold under a third of the bonded power, not %.3f", pair) + s.Require().GreaterOrEqualf(alone, 1.0/3, "val0 must hold at least a third of the bonded power, not %.3f", alone) + + s.T().Log("val2 and val3 refuse the consumer: two validators of four, under a third of the bonded power...") + s.setConsumerRefusalFrom(2, consumerID, true) + s.setConsumerRefusalFrom(3, consumerID, true) s.Require().Never(func() bool { - return s.queryProviderConsumerPhase(consumerID) != phaseLaunched + phase := s.queryProviderConsumerPhase(consumerID) + return phase != "" && phase != phaseLaunched }, 20*time.Second, 4*time.Second, - "a quarter of the bonded power refusing must not pause the consumer") + "two validators refusing with under a third of the bonded power must not pause the consumer") - s.T().Log("val1 refuses too: about half of the bonded power...") - s.setConsumerRefusalFrom(1, consumerID, true) + s.T().Log("val2 and val3 withdraw, and val0 refuses alone: one validator of four, half of the bonded power...") + s.setConsumerRefusalFrom(2, consumerID, false) + s.setConsumerRefusalFrom(3, consumerID, false) + s.setConsumerRefusalFrom(0, consumerID, true) s.Require().Eventuallyf(func() bool { return s.queryProviderConsumerPhase(consumerID) == phasePaused }, time.Minute, 2*time.Second, - "half of the bonded power refusing must pause the consumer") + "one validator refusing with half of the bonded power must pause the consumer") - s.T().Log("both withdraw their refusals and governance resumes the consumer...") + s.T().Log("val0 withdraws its refusal and governance resumes the consumer...") s.setConsumerRefusalFrom(0, consumerID, false) - s.setConsumerRefusalFrom(1, consumerID, false) - s.submitAndPassProposal(s.resumeProposalJSON(consumerID, "Resume consumer (refusals withdrawn)")) + s.submitAndPassProposal(s.resumeProposalJSON(consumerID, "Resume consumer (refusal withdrawn)")) s.Require().Eventuallyf(func() bool { return s.queryProviderConsumerPhase(consumerID) == phaseLaunched }, time.Minute, 2*time.Second, diff --git a/tests/e2e/e2e_refusal_and_deferral_test.go b/tests/e2e/e2e_refusal_and_deferral_test.go index 8842ee2a..4a00a3da 100644 --- a/tests/e2e/e2e_refusal_and_deferral_test.go +++ b/tests/e2e/e2e_refusal_and_deferral_test.go @@ -265,7 +265,7 @@ func (s *baseTestSuite) queryPendingEquivocations(consumerID string) ([]pendingE } // mustPendingEquivocations is queryPendingEquivocations for assertions. -func (s *baseTestSuite) mustPendingEquivocations(consumerID string) []pendingEquivocationJSON { +func (s *baseTestSuite) mustPendingEquivocations(consumerID string) []pendingEquivocationJSON { //nolint:unparam // the suites drive one consumer; the helper stays generic pending, err := s.queryPendingEquivocations(consumerID) s.Require().NoError(err) return pending diff --git a/tests/e2e/four_validator_helpers_test.go b/tests/e2e/four_validator_helpers_test.go index c4a586c0..c746d531 100644 --- a/tests/e2e/four_validator_helpers_test.go +++ b/tests/e2e/four_validator_helpers_test.go @@ -1,7 +1,6 @@ package e2e import ( - "fmt" "strconv" "time" @@ -25,7 +24,7 @@ const fourValidatorBlocksPerEpoch = 5 // downtime window is loosened, so a stopped consumer node never gets its // validator jailed on the provider, whose node keeps running. func (s *baseTestSuite) patchFourValidatorProviderGenesis(appState map[string]any) { - s.genesisSection(appState, "gov", "params")["voting_period"] = fmt.Sprintf("%ds", int64(s.cfg.govVotingPeriod/time.Second)) + s.setGovVotingPeriod(appState) params := s.genesisSection(appState, "vaasprovider", "params") params["blocks_per_epoch"] = strconv.Itoa(fourValidatorBlocksPerEpoch) diff --git a/tests/e2e/query_test.go b/tests/e2e/query_test.go index f9990f9d..02a276ed 100644 --- a/tests/e2e/query_test.go +++ b/tests/e2e/query_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "strconv" + "time" rpchttp "github.com/cometbft/cometbft/rpc/client/http" @@ -203,20 +204,35 @@ func (s *baseTestSuite) queryConsumerPreviousChainIDs(consumerID string) []previ // nextSequenceSend returns the sequence the provider's next IBC v2 packet over // the client takes; it moves exactly when the provider sends a packet over it. func (s *baseTestSuite) nextSequenceSend(clientID string) uint64 { + sequence, err := s.tryNextSequenceSend(clientID) + s.Require().NoErrorf(err, "failed to read the next send sequence of client %s", clientID) + return sequence +} + +// tryNextSequenceSend is nextSequenceSend returning an error instead of +// failing the test, for the conditions of Eventually and Never, which run in +// a goroutine of their own: a failure there stalls the wait to its timeout. +func (s *baseTestSuite) tryNextSequenceSend(clientID string) (uint64, error) { stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ providerBinary, "query", "ibc", "channelv2", "next-sequence-send", clientID, "--prove=false", "--home", providerHomePath, "--output", "json", }) - s.Require().NoErrorf(err, "failed to query the next send sequence of client %s: %s", clientID, stderr.String()) + if err != nil { + return 0, fmt.Errorf("querying the next send sequence of client %s: %w (stderr=%s)", clientID, err, stderr.String()) + } var res struct { NextSequenceSend string `json:"next_sequence_send"` } - s.Require().NoErrorf(json.Unmarshal(stdout.Bytes(), &res), "failed to decode the next send sequence of client %s: %s", clientID, stdout.String()) + if err := json.Unmarshal(stdout.Bytes(), &res); err != nil { + return 0, fmt.Errorf("decoding the next send sequence of client %s: %w (raw=%s)", clientID, err, stdout.String()) + } sequence, err := strconv.ParseUint(res.NextSequenceSend, 10, 64) - s.Require().NoErrorf(err, "next send sequence of client %s: %q", clientID, res.NextSequenceSend) - return sequence + if err != nil { + return 0, fmt.Errorf("next send sequence of client %s: %q: %w", clientID, res.NextSequenceSend, err) + } + return sequence, nil } // blockEvent is an event a block emitted outside any transaction, as the @@ -283,6 +299,26 @@ func blockEventMatches(ev blockEvent, want map[string]string) bool { return true } +// providerBlockTime returns the header time of the provider block at height: +// the block time the keepers took every deadline set or judged in that block +// from. +func (s *baseTestSuite) providerBlockTime(height int64) time.Time { + body, err := httpGet(fmt.Sprintf("%s/block?height=%d", s.providerRPCEndpoint(), height)) + s.Require().NoErrorf(err, "failed to read the provider's block %d", height) + var res struct { + Result struct { + Block struct { + Header struct { + Time time.Time `json:"time"` + } `json:"header"` + } `json:"block"` + } `json:"result"` + } + s.Require().NoErrorf(json.Unmarshal(body, &res), "failed to decode the provider's block %d: %s", height, body) + s.Require().Falsef(res.Result.Block.Header.Time.IsZero(), "the provider's block %d carries no time: %s", height, body) + return res.Result.Block.Header.Time +} + // queryProviderConsumerChains queries the provider for registered consumer chains. func (s *baseTestSuite) queryProviderConsumerChains() (string, error) { stdout, _, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ diff --git a/tests/e2e/validator_identity_helpers_test.go b/tests/e2e/validator_identity_helpers_test.go index 64e331a0..8d749b41 100644 --- a/tests/e2e/validator_identity_helpers_test.go +++ b/tests/e2e/validator_identity_helpers_test.go @@ -152,6 +152,31 @@ func consAddrFromBase64(b64 string) (string, error) { return sdk.ConsAddress(raw).String(), nil } +// tryStakingValidatorState is stakingValidatorState returning an error +// instead of failing the test, for the conditions of Eventually and Never, +// which run in a goroutine of their own: a failure there stalls the wait to +// its timeout. +func (s *baseTestSuite) tryStakingValidatorState(valoper string) (jailed bool, tokens string, err error) { + stdout, stderr, err := s.dockerExec(s.providerValRes[0].Container.ID, []string{ + providerBinary, "query", "staking", "validator", valoper, + "--home", providerHomePath, + "--output", "json", + }) + if err != nil { + return false, "", fmt.Errorf("querying staking validator %s: %w (stderr=%s)", valoper, err, stderr.String()) + } + var res struct { + Validator struct { + Jailed bool `json:"jailed"` + Tokens string `json:"tokens"` + } `json:"validator"` + } + if err := json.Unmarshal(stdout.Bytes(), &res); err != nil { + return false, "", fmt.Errorf("decoding staking validator %s: %w (raw=%s)", valoper, err, stdout.String()) + } + return res.Validator.Jailed, res.Validator.Tokens, nil +} + // providerValidatorConsAddr returns the bech32 consensus address of the // provider validator with the given operator address, failing the test if it // cannot be resolved.