llm-proxy: add distributed lock for background sources sync (#51544)

Adds a distributed lock, implemented by
[`redsync`](https://pkg.go.dev/github.com/go-redsync/redsync/v4), to the
background sources sync so that if we scale up to 10 Cloud Run LLM-proxy
instances they won't all do the same potentially-costly exhaustive sync
work which ultimately writes to a shared Redis instance.

The lock expires fast, such that the holder must regularly extend the
lock, and each instance will regularly try to claim the lock if they
don't hold it yet.

Addresses
https://github.com/sourcegraph/sourcegraph/pull/51534#discussion_r1186523143

## Test plan

The integration contention test passes consistently:

```
go test -v -count=100 -run ^TestSourcesWorkers$ github.com/sourcegraph/sourcegraph/enterprise/cmd/llm-proxy/internal/actor
```

The tests cover:

1. If an instance holds the lock, it will be the only instance doing
work
2. If an instance stops working, another instance will claim the lock
and start doing the work
This commit is contained in:
Robert Lin 2023-05-08 15:24:56 -07:00 committed by GitHub
parent 467dab1b88
commit 042a50482d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 245 additions and 33 deletions

View File

@ -895,6 +895,20 @@ def go_dependencies():
sum = "h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=",
version = "v1.0.0",
)
go_repository(
name = "com_github_bsm_ginkgo_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/bsm/ginkgo/v2",
sum = "h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ=",
version = "v2.5.0",
)
go_repository(
name = "com_github_bsm_gomega",
build_file_proto_mode = "disable_global",
importpath = "github.com/bsm/gomega",
sum = "h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8=",
version = "v1.20.0",
)
go_repository(
name = "com_github_bufbuild_buf",
@ -1727,8 +1741,8 @@ def go_dependencies():
name = "com_github_dgraph_io_ristretto",
build_file_proto_mode = "disable_global",
importpath = "github.com/dgraph-io/ristretto",
sum = "h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=",
version = "v0.1.0",
sum = "h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=",
version = "v0.1.1",
)
go_repository(
@ -2668,9 +2682,17 @@ def go_dependencies():
name = "com_github_go_redis_redis",
build_file_proto_mode = "disable_global",
importpath = "github.com/go-redis/redis",
sum = "h1:BKZuG6mCnRj5AOaWJXoCgf6rqTYnYJLe4en2hxT7r9o=",
version = "v6.15.8+incompatible",
sum = "h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=",
version = "v6.15.9+incompatible",
)
go_repository(
name = "com_github_go_redis_redis_v7",
build_file_proto_mode = "disable_global",
importpath = "github.com/go-redis/redis/v7",
sum = "h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4=",
version = "v7.4.0",
)
go_repository(
name = "com_github_go_redis_redis_v8",
build_file_proto_mode = "disable_global",
@ -2678,6 +2700,13 @@ def go_dependencies():
sum = "h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=",
version = "v8.11.5",
)
go_repository(
name = "com_github_go_redsync_redsync_v4",
build_file_proto_mode = "disable_global",
importpath = "github.com/go-redsync/redsync/v4",
sum = "h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM=",
version = "v4.8.1",
)
go_repository(
name = "com_github_go_resty_resty_v2",
@ -5796,6 +5825,13 @@ def go_dependencies():
sum = "h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ=",
version = "v0.0.0-20150907023854-cb7f23ec59be",
)
go_repository(
name = "com_github_redis_go_redis_v9",
build_file_proto_mode = "disable_global",
importpath = "github.com/redis/go-redis/v9",
sum = "h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE=",
version = "v9.0.2",
)
go_repository(
name = "com_github_rhnvrm_simples3",
@ -6411,6 +6447,14 @@ def go_dependencies():
sum = "h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=",
version = "v1.8.2",
)
go_repository(
name = "com_github_stvp_tempredis",
build_file_proto_mode = "disable_global",
importpath = "github.com/stvp/tempredis",
sum = "h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM=",
version = "v0.0.0-20181119212430-b82af8480203",
)
go_repository(
name = "com_github_subosito_gotenv",
build_file_proto_mode = "disable_global",
@ -7921,15 +7965,15 @@ def go_dependencies():
)
go_repository(
name = "com_google_cloud_go_trace",
build_file_proto_mode = "disable_global",
importpath = "cloud.google.com/go/trace",
sum = "h1:GFPLxbp5/FzdgTzor3nlNYNxMd6hLmzkE7sA9F0qQcA=",
version = "v1.8.0",
build_directives = [
# @go_googleapis is the modern version of @org_golang_google_genproto
# use @go_googleapis to avoid dependency conflicts between the two
"gazelle:resolve go google.golang.org/genproto/googleapis/api/annotations @go_googleapis//google/api:annotations_go_proto", # keep
],
build_file_proto_mode = "disable_global",
importpath = "cloud.google.com/go/trace",
sum = "h1:GFPLxbp5/FzdgTzor3nlNYNxMd6hLmzkE7sA9F0qQcA=",
version = "v1.8.0",
)
go_repository(
name = "com_google_cloud_go_translate",

View File

@ -12,6 +12,7 @@ go_library(
"//enterprise/cmd/llm-proxy/internal/limiter",
"//internal/goroutine",
"//lib/errors",
"@com_github_go_redsync_redsync_v4//:redsync",
"@com_github_sourcegraph_conc//pool",
],
)
@ -20,8 +21,18 @@ go_test(
name = "actor_test",
srcs = ["source_test.go"],
embed = [":actor"],
tags = [
# Test requires localhost database
"requires-network",
],
deps = [
"//internal/redispool",
"//lib/errors",
"@com_github_go_redsync_redsync_v4//:redsync",
"@com_github_go_redsync_redsync_v4//redis/redigo",
"@com_github_gomodule_redigo//redis",
"@com_github_sourcegraph_conc//:conc",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
],
)

View File

@ -4,6 +4,7 @@ import (
"context"
"time"
"github.com/go-redsync/redsync/v4"
"github.com/sourcegraph/conc/pool"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
@ -55,22 +56,73 @@ func (s Sources) Get(ctx context.Context, token string) (*Actor, error) {
}
// Worker is a goroutine.BackgroundRoutine that runs any SourceSyncer implementations
// at a regular interval.
func (s Sources) Worker(rootInterval time.Duration) goroutine.BackgroundRoutine {
return goroutine.NewPeriodicGoroutine(
context.Background(),
"sources", "sources sync worker",
rootInterval,
&sourcesPeriodicHandler{sources: s})
// at a regular interval. It uses a redsync.Mutex to ensure only one worker is running
// at a time.
func (s Sources) Worker(rmux *redsync.Mutex, rootInterval time.Duration) goroutine.BackgroundRoutine {
return &redisLockedBackgroundRoutine{
rmux: rmux,
routine: goroutine.NewPeriodicGoroutine(
context.Background(),
"sources", "sources sync worker",
rootInterval,
&sourcesPeriodicHandler{
rmux: rmux,
sources: s,
}),
}
}
// redisLockedBackgroundRoutine attempts to acquire a redsync lock before starting,
// and releases it when stopped.
type redisLockedBackgroundRoutine struct {
rmux *redsync.Mutex
routine goroutine.BackgroundRoutine
}
func (s *redisLockedBackgroundRoutine) Start() {
// Best-effort attempt to acquire lock immediately.
// We check if we have the lock first because in tests we may manually acquire
// it first to keep tests stable.
if expire := s.rmux.Until(); expire.IsZero() {
_ = s.rmux.LockContext(context.Background())
}
s.routine.Start()
}
func (s *redisLockedBackgroundRoutine) Stop() {
s.routine.Stop()
// If we have the lock, release it and let somebody else work
if expire := s.rmux.Until(); !expire.IsZero() {
s.rmux.Unlock()
}
}
// sourcesPeriodicHandler is a handler for NewPeriodicGoroutine
type sourcesPeriodicHandler struct {
rmux *redsync.Mutex
sources Sources
}
var _ goroutine.Handler = &sourcesPeriodicHandler{}
func (s *sourcesPeriodicHandler) Handle(ctx context.Context) error {
// If we are not holding a lock, try to acquire it.
if expire := s.rmux.Until(); expire.IsZero() {
// If another instance is working on background syncs, we don't want to
// do anything. We should check every time still in case the current worker
// goes offline, we want to be ready to pick up the work.
if err := s.rmux.LockContext(ctx); errors.Is(err, redsync.ErrFailed) {
return nil // ignore lock contention errors
} else if err != nil {
return errors.Wrap(err, "acquire worker lock")
}
} else {
// Otherwise, extend our lock so that we can keep working.
_, _ = s.rmux.ExtendContext(ctx)
}
p := pool.New().WithErrors().WithContext(ctx)
for _, src := range s.sources {
if src, ok := src.(SourceSyncer); ok {

View File

@ -2,11 +2,19 @@ package actor
import (
"context"
"strconv"
"testing"
"time"
"github.com/go-redsync/redsync/v4"
"github.com/go-redsync/redsync/v4/redis/redigo"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/conc"
"github.com/sourcegraph/sourcegraph/internal/redispool"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -27,25 +35,79 @@ func (m *mockSourceSyncer) Sync(context.Context) error {
return nil
}
func TestSourcesWorker(t *testing.T) {
var s mockSourceSyncer
w := (Sources{&s}).Worker(time.Millisecond)
stopped := make(chan struct{})
func TestSourcesWorkers(t *testing.T) {
// Connect to local redis for testing, this is the same URL used in rcache.SetupForTest
p, ok := redispool.NewKeyValue("127.0.0.1:6379", &redis.Pool{
MaxIdle: 3,
IdleTimeout: 5 * time.Second,
}).Pool()
if !ok {
t.Fatal("real redis is required")
}
rs := redsync.New(redigo.NewPool(p))
// Work happens after start
go func() {
// Randomized lock name to avoid flakiness when running with count>1
lockName := t.Name() + strconv.Itoa(time.Now().Nanosecond())
// Run workers in group to ensure cleanup
g := conc.NewWaitGroup()
// Start first worker, aquiromg tje mutex first for test stability
sourceWorkerMutex1 := rs.NewMutex(lockName)
require.NoError(t, sourceWorkerMutex1.Lock())
s1 := &mockSourceSyncer{}
stop1 := make(chan struct{})
g.Go(func() {
w := (Sources{s1}).Worker(sourceWorkerMutex1, time.Millisecond)
go func() {
<-stop1
w.Stop()
}()
w.Start()
stopped <- struct{}{}
}()
time.Sleep(9 * time.Millisecond)
assert.NotZero(t, s.syncCount)
})
// No work happens after stop
w.Stop()
count := s.syncCount
time.Sleep(10 * time.Millisecond)
assert.LessOrEqual(t, count, s.syncCount)
// Start second worker to compete with first worker
s2 := &mockSourceSyncer{}
stop2 := make(chan struct{})
g.Go(func() {
sourceWorkerMutex := rs.NewMutex(lockName,
// Competing worker should only try once to avoid getting stuck
redsync.WithTries(1))
w := (Sources{s2}).Worker(sourceWorkerMutex, time.Millisecond)
go func() {
<-stop2
w.Stop()
}()
w.Start()
})
println("waiting for stop")
<-stopped
// Wait for some things to happen
time.Sleep(100 * time.Millisecond)
t.Run("only the first worker should be doing work", func(t *testing.T) {
assert.NotZero(t, s1.syncCount)
assert.Zero(t, s2.syncCount)
})
// Stop the first worker and wait a bit
close(stop1)
count1 := s1.syncCount // Save the count to assert later
time.Sleep(100 * time.Millisecond)
t.Run("first worker does no work after stop", func(t *testing.T) {
// Bounded range assertion to avoid flakiness
assert.GreaterOrEqual(t, count1, s1.syncCount-1)
assert.LessOrEqual(t, count1, s1.syncCount+1)
})
// Worker 2 should pick up work
t.Run("second worker does work after first worker stops", func(t *testing.T) {
assert.NotZero(t, s2.syncCount)
})
// Stop worker 2
close(stop2)
// Wait for everyone to go home for the weekend
g.Wait()
}

View File

@ -30,6 +30,8 @@ go_library(
"//internal/service",
"//internal/version",
"//lib/errors",
"@com_github_go_redsync_redsync_v4//:redsync",
"@com_github_go_redsync_redsync_v4//redis/redigo",
"@com_github_googlecloudplatform_opentelemetry_operations_go_exporter_trace//:trace",
"@com_github_gorilla_mux//:mux",
"@com_github_sourcegraph_log//:log",

View File

@ -9,6 +9,9 @@ import (
"github.com/gorilla/mux"
"github.com/sourcegraph/log"
"github.com/go-redsync/redsync/v4"
"github.com/go-redsync/redsync/v4/redis/redigo"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
@ -64,6 +67,21 @@ func Main(ctx context.Context, obctx *observation.Context, ready service.ReadyFu
Handler: handler,
})
// Set up redis-based distributed mutex for the source syncer worker
p, ok := redispool.Store.Pool()
if !ok {
return errors.New("real redis is required")
}
sourceWorkerMutex := redsync.New(redigo.NewPool(p)).NewMutex("source-syncer-worker",
// Do not retry endlessly becuase it's very likely that someone else has
// a long-standing hold on the mutex. We will try again on the next periodic
// goroutine run.
redsync.WithTries(1),
// Expire locks at 2x sync interval to avoid contention while avoiding
// the lock getting stuck for too long if something happens. Every handler
// iteration, we will extend the lock.
redsync.WithExpiry(2*config.SourcesSyncInterval))
// Mark health server as ready and go!
ready()
obctx.Logger.Info("service ready", log.String("address", config.Address))
@ -71,7 +89,7 @@ func Main(ctx context.Context, obctx *observation.Context, ready service.ReadyFu
// Block until done
goroutine.MonitorBackgroundRoutines(ctx,
server,
sources.Worker(config.SourcesSyncInterval))
sources.Worker(sourceWorkerMutex, config.SourcesSyncInterval))
return nil
}

4
go.mod
View File

@ -263,6 +263,8 @@ require (
sigs.k8s.io/yaml v1.3.0
)
require github.com/go-redsync/redsync/v4 v4.8.1
require (
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/trace v1.8.0 // indirect
@ -291,6 +293,8 @@ require (
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc // indirect
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20210128111500-3ff779b52992 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect

19
go.sum
View File

@ -374,6 +374,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.6.0 h1:knToPYa2xtfg42U3I6punFEjaGFKWQRXJwj
github.com/bradleyjkemp/cupaloy/v2 v2.6.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
github.com/bufbuild/buf v1.4.0 h1:GqE3a8CMmcFvWPzuY3Mahf9Kf3S9XgZ/ORpfYFzO+90=
github.com/bufbuild/buf v1.4.0/go.mod h1:mwHG7klTHnX+rM/ym8LXGl7vYpVmnwT96xWoRB4H5QI=
github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
@ -637,6 +639,8 @@ github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
@ -955,6 +959,14 @@ github.com/go-openapi/validate v0.22.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUri
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4=
github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redsync/redsync/v4 v4.8.1 h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM=
github.com/go-redsync/redsync/v4 v4.8.1/go.mod h1:LmUAsQuQxhzZAoGY7JS6+dNhNmZyonMZiiEDY9plotM=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
@ -1250,6 +1262,7 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN
github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@ -1265,6 +1278,7 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP
github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
github.com/hashicorp/go-retryablehttp v0.5.1/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
@ -1758,6 +1772,7 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@ -1905,6 +1920,8 @@ github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yRE
github.com/rafaeljusto/redigomock/v3 v3.1.2 h1:B4Y0XJQiPjpwYmkH55aratKX1VfR+JRqzmDKyZbC99o=
github.com/rafaeljusto/redigomock/v3 v3.1.2/go.mod h1:F9zPqz8rMriScZkPtUiLJoLruYcpGo/XXREpeyasREM=
github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q=
github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE=
github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps=
github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@ -2098,6 +2115,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM=
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=