diff --git a/CHANGELOG.md b/CHANGELOG.md index 52171fcf..7150efdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Periodic jobs with IDs may now be removed by ID using the new `PeriodicJobBundle.RemoveByID` and `PeriodicJobBundle.RemoveManyByID`. [PR #1071](https://github.com/riverqueue/river/pull/1071). +### Changed + +- Decrease `serviceutil.MaxAttemptsBeforeResetDefault` from 10 to 7, lowering the effective limit on most internal exponential backoffs from ~512 seconds to 64 seconds. Further lowered the leader elector's keep leadership backoff interval to cap out at 4 seconds since leadership without a successful heartbeat will be lost soon after that anyway. [PR #1079](https://github.com/riverqueue/river/pull/1079). + ### Fixed - Fix snoozed events emitted from `rivertest.Worker` when snooze duration is zero seconds. [PR #1057](https://github.com/riverqueue/river/pull/1057). diff --git a/internal/leadership/elector.go b/internal/leadership/elector.go index 55666b2c..7d17337c 100644 --- a/internal/leadership/elector.go +++ b/internal/leadership/elector.go @@ -349,7 +349,7 @@ func (e *Elector) keepLeadershipLoop(ctx context.Context) error { return err } - sleepDuration := serviceutil.ExponentialBackoff(numErrors, serviceutil.MaxAttemptsBeforeResetDefault) + sleepDuration := serviceutil.ExponentialBackoff(numErrors, 3) e.Logger.ErrorContext(ctx, e.Name+": Error attempting reelection", e.errorSlogArgs(err, numErrors, sleepDuration)...) serviceutil.CancellableSleep(ctx, sleepDuration) continue @@ -387,7 +387,7 @@ func (e *Elector) attemptResignLoop(ctx context.Context) { for attempt := 1; attempt <= maxNumErrors; attempt++ { if err := e.attemptResign(ctx, attempt); err != nil { - sleepDuration := serviceutil.ExponentialBackoff(attempt, serviceutil.MaxAttemptsBeforeResetDefault) + sleepDuration := serviceutil.ExponentialBackoff(attempt, maxNumErrors) e.Logger.ErrorContext(ctx, e.Name+": Error attempting to resign", e.errorSlogArgs(err, attempt, sleepDuration)...) serviceutil.CancellableSleep(ctx, sleepDuration) diff --git a/rivershared/util/serviceutil/service_util.go b/rivershared/util/serviceutil/service_util.go index 164e3fe5..dde3bbe6 100644 --- a/rivershared/util/serviceutil/service_util.go +++ b/rivershared/util/serviceutil/service_util.go @@ -44,7 +44,9 @@ func CancellableSleepC(ctx context.Context, sleepDuration time.Duration) <-chan // into a ridiculously distant future. This constant is typically injected into // the CancellableSleepExponentialBackoff function. It could technically take // another value instead, but shouldn't unless there's a good reason to do so. -const MaxAttemptsBeforeResetDefault = 10 +// +// The value of 7 corresponds to a max sleep duration of 64 seconds ±10% jitter. +const MaxAttemptsBeforeResetDefault = 7 // ExponentialBackoff returns a duration for a reasonable exponential backoff // interval for a service based on the given attempt number, which can then be @@ -52,6 +54,15 @@ const MaxAttemptsBeforeResetDefault = 10 // +/- 10% random jitter. Sleep is cancelled if the given context is cancelled. // // Attempt should start at one for the first backoff/failure. +// +// Backoff values when using MaxAttemptsBeforeResetDefault are: +// - 1s +// - 2s +// - 4s +// - 8s +// - 16s +// - 32s +// - 64s func ExponentialBackoff(attempt, maxAttemptsBeforeReset int) time.Duration { retrySeconds := exponentialBackoffSecondsWithoutJitter(attempt, maxAttemptsBeforeReset) diff --git a/rivershared/util/serviceutil/service_util_test.go b/rivershared/util/serviceutil/service_util_test.go index fa4bc730..8b991d35 100644 --- a/rivershared/util/serviceutil/service_util_test.go +++ b/rivershared/util/serviceutil/service_util_test.go @@ -77,6 +77,8 @@ func TestExponentialBackoff(t *testing.T) { require.InDelta(t, 8.0, ExponentialBackoff(4, MaxAttemptsBeforeResetDefault).Seconds(), 8.0*0.1) require.InDelta(t, 16.0, ExponentialBackoff(5, MaxAttemptsBeforeResetDefault).Seconds(), 16.0*0.1) require.InDelta(t, 32.0, ExponentialBackoff(6, MaxAttemptsBeforeResetDefault).Seconds(), 32.0*0.1) + require.InDelta(t, 64.0, ExponentialBackoff(7, MaxAttemptsBeforeResetDefault).Seconds(), 64.0*0.1) + require.InDelta(t, 1.0, ExponentialBackoff(8, MaxAttemptsBeforeResetDefault).Seconds(), 1.0*0.1) } func TestExponentialBackoffSecondsWithoutJitter(t *testing.T) { @@ -89,8 +91,5 @@ func TestExponentialBackoffSecondsWithoutJitter(t *testing.T) { require.Equal(t, 16, int(exponentialBackoffSecondsWithoutJitter(5, MaxAttemptsBeforeResetDefault))) require.Equal(t, 32, int(exponentialBackoffSecondsWithoutJitter(6, MaxAttemptsBeforeResetDefault))) require.Equal(t, 64, int(exponentialBackoffSecondsWithoutJitter(7, MaxAttemptsBeforeResetDefault))) - require.Equal(t, 128, int(exponentialBackoffSecondsWithoutJitter(8, MaxAttemptsBeforeResetDefault))) - require.Equal(t, 256, int(exponentialBackoffSecondsWithoutJitter(9, MaxAttemptsBeforeResetDefault))) - require.Equal(t, 512, int(exponentialBackoffSecondsWithoutJitter(10, MaxAttemptsBeforeResetDefault))) - require.Equal(t, 1, int(exponentialBackoffSecondsWithoutJitter(11, MaxAttemptsBeforeResetDefault))) // resets + require.Equal(t, 1, int(exponentialBackoffSecondsWithoutJitter(8, MaxAttemptsBeforeResetDefault))) // resets }