From 2a3bd2b515900c88007b14f016b7de8ab33ddf3d Mon Sep 17 00:00:00 2001 From: Erik Seliger Date: Wed, 14 Apr 2021 15:06:15 +0200 Subject: [PATCH] Upgrade aws-sdk-go-v2 (#19155) This PR: - Migrates to the AWS v2 API in the codeintel s3 adapter - Upgrades the v2 package to the newest version --- cmd/server/Dockerfile | 2 +- dev/go-mod-update.sh | 3 - doc/dependency_decisions.yml | 7 + docker-images/minio/build.sh | 4 +- .../cmd/precise-code-intel-worker/main.go | 15 +- .../stores/uploadstore/config_test.go | 68 ++++++-- .../stores/uploadstore/mock_s3_api_test.go | 25 ++- .../codeintel/stores/uploadstore/s3_api.go | 32 ++-- .../codeintel/stores/uploadstore/s3_client.go | 153 +++++++++--------- .../stores/uploadstore/s3_client_test.go | 70 ++++---- go.mod | 10 +- go.sum | 35 +++- internal/extsvc/awscodecommit/client.go | 15 +- internal/extsvc/awscodecommit/codehost.go | 6 +- internal/extsvc/awscodecommit/repos.go | 23 ++- internal/repos/awscodecommit.go | 63 ++++---- 16 files changed, 289 insertions(+), 242 deletions(-) diff --git a/cmd/server/Dockerfile b/cmd/server/Dockerfile index 574818c717a..d78d4c511fc 100644 --- a/cmd/server/Dockerfile +++ b/cmd/server/Dockerfile @@ -54,7 +54,7 @@ COPY --from=comby/comby:0.18.4@sha256:b47ce282778bfea7f80d45f5ef0cc546ba0d6347ba COPY --from=sourcegraph/syntect_server:331beda@sha256:6b8950b41993af3d10300b5a160fab1c06c337cb4614978f4fdb76e2588afbfe /syntect_server /usr/local/bin/ # install minio (keep this up to date with docker-images/minio/Dockerfile) -ENV MINIO_VERSION=RELEASE.2020-10-18T21-54-12Z +ENV MINIO_VERSION=RELEASE.2021-04-06T23-11-00Z RUN wget "https://dl.min.io/server/minio/release/linux-amd64/archive/minio.$MINIO_VERSION" && \ chmod +x "minio.$MINIO_VERSION" && \ mv "minio.$MINIO_VERSION" /usr/local/bin/minio diff --git a/dev/go-mod-update.sh b/dev/go-mod-update.sh index 40324b3b29f..c5935f41a63 100755 --- a/dev/go-mod-update.sh +++ b/dev/go-mod-update.sh @@ -14,9 +14,6 @@ go get -u all ## names. This is causing our use of our k8s client to panic. go get github.com/golang/protobuf@v1.3.5 -## Newer versions removed some types in the endpoint package we relied on. Unsure how to fix yet, so punting. -go get github.com/aws/aws-sdk-go-v2@v0.20.0 - # Cleanup and validate everything still works go mod tidy go test -short -failfast ./... >/dev/null diff --git a/doc/dependency_decisions.yml b/doc/dependency_decisions.yml index a5eccd38eb2..a769f698cca 100644 --- a/doc/dependency_decisions.yml +++ b/doc/dependency_decisions.yml @@ -346,3 +346,10 @@ :why: Internal module :versions: [] :when: 2021-03-09 19:42:12.214862934 Z +- - :license + - github.com/aws/smithy-go + - Apache 2.0 + - :who: + :why: Inference broken, LICENSE file lives at https://github.com/aws/smithy-go/blob/main/LICENSE + :versions: [] + :when: 2021-04-14 12:51:19.503802000 Z diff --git a/docker-images/minio/build.sh b/docker-images/minio/build.sh index 20f88d890e3..5de0d1d8e6d 100755 --- a/docker-images/minio/build.sh +++ b/docker-images/minio/build.sh @@ -3,7 +3,7 @@ set -ex cd "$(dirname "${BASH_SOURCE[0]}")" -# Retag the 2020-10-18 MinIO release -MINIO_RELEASE="RELEASE.2020-10-18T21-54-12Z" +# Retag the 2021-04-06 MinIO release +MINIO_RELEASE="RELEASE.2021-04-06T23-11-00Z" docker pull minio/minio:$MINIO_RELEASE docker tag minio/minio:$MINIO_RELEASE "$IMAGE" diff --git a/enterprise/cmd/precise-code-intel-worker/main.go b/enterprise/cmd/precise-code-intel-worker/main.go index 318f73ec585..bfc33bb6fca 100644 --- a/enterprise/cmd/precise-code-intel-worker/main.go +++ b/enterprise/cmd/precise-code-intel-worker/main.go @@ -8,7 +8,7 @@ import ( "net/http" "time" - "github.com/aws/aws-sdk-go/aws/awserr" + smithyhttp "github.com/aws/smithy-go/transport/http" "github.com/inconshreveable/log15" "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" @@ -199,15 +199,6 @@ func initializeUploadStore(ctx context.Context, uploadStore uploadstore.Store) e } func isRequestError(err error) bool { - for err != nil { - if e, ok := err.(awserr.Error); ok { - if e.Code() == "RequestError" { - return true - } - } - - err = errors.Unwrap(err) - } - - return false + var rse *smithyhttp.RequestSendError + return errors.As(err, &rse) } diff --git a/enterprise/internal/codeintel/stores/uploadstore/config_test.go b/enterprise/internal/codeintel/stores/uploadstore/config_test.go index 6dd184a0ea3..29f9f05013f 100644 --- a/enterprise/internal/codeintel/stores/uploadstore/config_test.go +++ b/enterprise/internal/codeintel/stores/uploadstore/config_test.go @@ -1,8 +1,13 @@ package uploadstore import ( + "context" "testing" "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/go-cmp/cmp" ) func TestConfigDefaults(t *testing.T) { @@ -70,25 +75,44 @@ func TestConfigS3(t *testing.T) { } } -func TestConfigMinIOSessionOptions(t *testing.T) { +func TestS3ClientOptions(t *testing.T) { config := Config{} config.SetMockGetter(mapGetter(nil)) config.Load() - options := s3SessionOptions("minio", config.S3) + // minIO + { + options := &s3.Options{} + s3ClientOptions("minio", config.S3)(options) - if value := *options.Config.Region; value != "us-east-1" { - t.Errorf("unexpected region. want=%s have=%s", "us-east-1", value) + if options.EndpointResolver == nil { + t.Fatalf("unexpected endpoint option") + } + endpoint, err := options.EndpointResolver.ResolveEndpoint("us-east-2", s3.EndpointResolverOptions{}) + if err != nil { + t.Fatal(err) + } + if endpoint.URL != "http://minio:9000" { + t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", endpoint.URL) + } + + if !options.UsePathStyle { + t.Errorf("invalid UsePathStyle setting for S3Options") + } } - if value := *options.Config.Endpoint; value != "http://minio:9000" { - t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", value) - } - if options.Config.S3ForcePathStyle == nil || !*options.Config.S3ForcePathStyle { - t.Errorf("expected path style option") + + // S3 + { + options := &s3.Options{} + s3ClientOptions("s3", config.S3)(options) + + if diff := cmp.Diff(&s3.Options{}, options); diff != "" { + t.Fatalf("invalid s3 options returned for S3: %s", diff) + } } } -func TestConfigS3SessionOptions(t *testing.T) { +func TestS3ClientConfig(t *testing.T) { env := map[string]string{ "PRECISE_CODE_INTEL_UPLOAD_BACKEND": "S3", "PRECISE_CODE_INTEL_UPLOAD_BUCKET": "lsif-uploads", @@ -104,16 +128,28 @@ func TestConfigS3SessionOptions(t *testing.T) { config.SetMockGetter(mapGetter(env)) config.Load() - options := s3SessionOptions("s3", config.S3) + cfg, err := s3ClientConfig(context.Background(), config.S3) + if err != nil { + t.Fatal(err) + } - if value := *options.Config.Region; value != "us-east-2" { + if value := cfg.Region; value != "us-east-2" { t.Errorf("unexpected region. want=%s have=%s", "us-east-2", value) } - if options.Config.Endpoint != nil { - t.Errorf("unexpected endpoint option") + cred, err := cfg.Credentials.Retrieve(context.Background()) + if err != nil { + t.Fatal(err) } - if options.Config.S3ForcePathStyle != nil { - t.Errorf("unexpected path style option") + if diff := cmp.Diff(aws.Credentials{ + AccessKeyID: config.S3.AccessKeyID, + SecretAccessKey: config.S3.SecretAccessKey, + SessionToken: config.S3.SessionToken, + Source: "StaticCredentials", + }, cred); diff != "" { + t.Errorf("invalid credential returned: %s", diff) + } + if cfg.EndpointResolver != nil { + t.Errorf("unexpected endpoint option") } } diff --git a/enterprise/internal/codeintel/stores/uploadstore/mock_s3_api_test.go b/enterprise/internal/codeintel/stores/uploadstore/mock_s3_api_test.go index fb03c93bca5..2c6e6d69a7b 100644 --- a/enterprise/internal/codeintel/stores/uploadstore/mock_s3_api_test.go +++ b/enterprise/internal/codeintel/stores/uploadstore/mock_s3_api_test.go @@ -6,8 +6,7 @@ import ( "context" "sync" - s3 "github.com/aws/aws-sdk-go/service/s3" - s3manager "github.com/aws/aws-sdk-go/service/s3/s3manager" + s3 "github.com/aws/aws-sdk-go-v2/service/s3" ) // MockS3API is a mock implementation of the s3API interface (from the @@ -1143,7 +1142,7 @@ type MockS3Uploader struct { func NewMockS3Uploader() *MockS3Uploader { return &MockS3Uploader{ UploadFunc: &S3UploaderUploadFunc{ - defaultHook: func(context.Context, *s3manager.UploadInput) error { + defaultHook: func(context.Context, *s3.PutObjectInput) error { return nil }, }, @@ -1155,7 +1154,7 @@ func NewMockS3Uploader() *MockS3Uploader { // github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/uploadstore). // It is redefined here as it is unexported in the source packge. type surrogateMockS3Uploader interface { - Upload(context.Context, *s3manager.UploadInput) error + Upload(context.Context, *s3.PutObjectInput) error } // NewMockS3UploaderFrom creates a new mock of the MockS3Uploader interface. @@ -1171,15 +1170,15 @@ func NewMockS3UploaderFrom(i surrogateMockS3Uploader) *MockS3Uploader { // S3UploaderUploadFunc describes the behavior when the Upload method of the // parent MockS3Uploader instance is invoked. type S3UploaderUploadFunc struct { - defaultHook func(context.Context, *s3manager.UploadInput) error - hooks []func(context.Context, *s3manager.UploadInput) error + defaultHook func(context.Context, *s3.PutObjectInput) error + hooks []func(context.Context, *s3.PutObjectInput) error history []S3UploaderUploadFuncCall mutex sync.Mutex } // Upload delegates to the next hook function in the queue and stores the // parameter and result values of this invocation. -func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3manager.UploadInput) error { +func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3.PutObjectInput) error { r0 := m.UploadFunc.nextHook()(v0, v1) m.UploadFunc.appendCall(S3UploaderUploadFuncCall{v0, v1, r0}) return r0 @@ -1187,7 +1186,7 @@ func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3manager.UploadInput) e // SetDefaultHook sets function that is called when the Upload method of the // parent MockS3Uploader instance is invoked and the hook queue is empty. -func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3manager.UploadInput) error) { +func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3.PutObjectInput) error) { f.defaultHook = hook } @@ -1195,7 +1194,7 @@ func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3mana // Upload method of the parent MockS3Uploader instance invokes the hook at // the front of the queue and discards it. After the queue is empty, the // default hook function is invoked for any future action. -func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3manager.UploadInput) error) { +func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3.PutObjectInput) error) { f.mutex.Lock() f.hooks = append(f.hooks, hook) f.mutex.Unlock() @@ -1204,7 +1203,7 @@ func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3manager.Up // SetDefaultReturn calls SetDefaultDefaultHook with a function that returns // the given values. func (f *S3UploaderUploadFunc) SetDefaultReturn(r0 error) { - f.SetDefaultHook(func(context.Context, *s3manager.UploadInput) error { + f.SetDefaultHook(func(context.Context, *s3.PutObjectInput) error { return r0 }) } @@ -1212,12 +1211,12 @@ func (f *S3UploaderUploadFunc) SetDefaultReturn(r0 error) { // PushReturn calls PushDefaultHook with a function that returns the given // values. func (f *S3UploaderUploadFunc) PushReturn(r0 error) { - f.PushHook(func(context.Context, *s3manager.UploadInput) error { + f.PushHook(func(context.Context, *s3.PutObjectInput) error { return r0 }) } -func (f *S3UploaderUploadFunc) nextHook() func(context.Context, *s3manager.UploadInput) error { +func (f *S3UploaderUploadFunc) nextHook() func(context.Context, *s3.PutObjectInput) error { f.mutex.Lock() defer f.mutex.Unlock() @@ -1255,7 +1254,7 @@ type S3UploaderUploadFuncCall struct { Arg0 context.Context // Arg1 is the value of the 2nd argument passed to this method // invocation. - Arg1 *s3manager.UploadInput + Arg1 *s3.PutObjectInput // Result0 is the value of the 1st result returned from this method // invocation. Result0 error diff --git a/enterprise/internal/codeintel/stores/uploadstore/s3_api.go b/enterprise/internal/codeintel/stores/uploadstore/s3_api.go index aa7f7ecb193..14fae86528c 100644 --- a/enterprise/internal/codeintel/stores/uploadstore/s3_api.go +++ b/enterprise/internal/codeintel/stores/uploadstore/s3_api.go @@ -3,8 +3,8 @@ package uploadstore import ( "context" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" ) type s3API interface { @@ -20,52 +20,52 @@ type s3API interface { } type s3Uploader interface { - Upload(ctx context.Context, input *s3manager.UploadInput) error + Upload(ctx context.Context, input *s3.PutObjectInput) error } -type s3APIShim struct{ *s3.S3 } -type s3UploaderShim struct{ *s3manager.Uploader } +type s3APIShim struct{ *s3.Client } +type s3UploaderShim struct{ *manager.Uploader } var _ s3API = &s3APIShim{} var _ s3Uploader = &s3UploaderShim{} func (s *s3APIShim) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) (*s3.CreateBucketOutput, error) { - return s.S3.CreateBucketWithContext(ctx, input) + return s.Client.CreateBucket(ctx, input) } func (s *s3APIShim) PutBucketLifecycleConfiguration(ctx context.Context, input *s3.PutBucketLifecycleConfigurationInput) (*s3.PutBucketLifecycleConfigurationOutput, error) { - return s.S3.PutBucketLifecycleConfigurationWithContext(ctx, input) + return s.Client.PutBucketLifecycleConfiguration(ctx, input) } func (s *s3APIShim) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { - return s.S3.HeadObjectWithContext(ctx, input) + return s.Client.HeadObject(ctx, input) } func (s *s3APIShim) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { - return s.S3.GetObjectWithContext(ctx, input) + return s.Client.GetObject(ctx, input) } func (s *s3APIShim) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { - return s.S3.DeleteObjectWithContext(ctx, input) + return s.Client.DeleteObject(ctx, input) } func (s *s3APIShim) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { - return s.S3.CreateMultipartUploadWithContext(ctx, input) + return s.Client.CreateMultipartUpload(ctx, input) } func (s *s3APIShim) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) { - return s.S3.AbortMultipartUploadWithContext(ctx, input) + return s.Client.AbortMultipartUpload(ctx, input) } func (s *s3APIShim) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { - return s.S3.UploadPartCopyWithContext(ctx, input) + return s.Client.UploadPartCopy(ctx, input) } func (s *s3APIShim) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { - return s.S3.CompleteMultipartUploadWithContext(ctx, input) + return s.Client.CompleteMultipartUpload(ctx, input) } -func (s *s3UploaderShim) Upload(ctx context.Context, input *s3manager.UploadInput) error { - _, err := s.Uploader.UploadWithContext(ctx, input) +func (s *s3UploaderShim) Upload(ctx context.Context, input *s3.PutObjectInput) error { + _, err := s.Uploader.Upload(ctx, input) return err } diff --git a/enterprise/internal/codeintel/stores/uploadstore/s3_client.go b/enterprise/internal/codeintel/stores/uploadstore/s3_client.go index 8f852facde0..a976c05da52 100644 --- a/enterprise/internal/codeintel/stores/uploadstore/s3_client.go +++ b/enterprise/internal/codeintel/stores/uploadstore/s3_client.go @@ -9,12 +9,12 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/inconshreveable/log15" "github.com/opentracing/opentracing-go/log" "github.com/pkg/errors" @@ -25,12 +25,12 @@ import ( ) type s3Store struct { - bucket string - ttl time.Duration - manageBucket bool - client s3API - uploader s3Uploader - operations *operations + bucket string + manageBucket bool + client s3API + uploader s3Uploader + bucketLifecycleConfiguration *s3types.BucketLifecycleConfiguration + operations *operations } var _ Store = &s3Store{} @@ -53,25 +53,25 @@ func (c *S3Config) load(parent *env.BaseConfig) { // newS3FromConfig creates a new store backed by AWS Simple Storage Service. func newS3FromConfig(ctx context.Context, config *Config, operations *operations) (Store, error) { - sess, err := session.NewSessionWithOptions(s3SessionOptions(config.Backend, config.S3)) + cfg, err := s3ClientConfig(ctx, config.S3) if err != nil { return nil, err } - s3Client := s3.New(sess) + s3Client := s3.NewFromConfig(cfg, s3ClientOptions(config.Backend, config.S3)) api := &s3APIShim{s3Client} - uploader := &s3UploaderShim{s3manager.NewUploaderWithClient(s3Client)} - return newS3WithClients(api, uploader, config.Bucket, config.TTL, config.ManageBucket, operations), nil + uploader := &s3UploaderShim{manager.NewUploader(s3Client)} + return newS3WithClients(api, uploader, config.Bucket, config.ManageBucket, s3BucketLifecycleConfiguration(config.Backend, config.TTL), operations), nil } -func newS3WithClients(client s3API, uploader s3Uploader, bucket string, ttl time.Duration, manageBucket bool, operations *operations) *s3Store { +func newS3WithClients(client s3API, uploader s3Uploader, bucket string, manageBucket bool, lifecycleConfiguration *s3types.BucketLifecycleConfiguration, operations *operations) *s3Store { return &s3Store{ - bucket: bucket, - ttl: ttl, - manageBucket: manageBucket, - client: client, - uploader: uploader, - operations: operations, + bucket: bucket, + manageBucket: manageBucket, + client: client, + uploader: uploader, + operations: operations, + bucketLifecycleConfiguration: lifecycleConfiguration, } } @@ -166,7 +166,7 @@ func (s *s3Store) Upload(ctx context.Context, key string, r io.Reader) (_ int64, cr := &countingReader{r: r} - if err := s.uploader.Upload(ctx, &s3manager.UploadInput{ + if err := s.uploader.Upload(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), Body: cr, @@ -220,7 +220,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st Bucket: multipartUpload.Bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId, - PartNumber: aws.Int64(int64(partNumber)), + PartNumber: int32(partNumber), CopySource: aws.String(fmt.Sprintf("%s/%s", s.bucket, source)), }) if err != nil { @@ -236,13 +236,13 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st return 0, err } - var parts []*s3.CompletedPart + var parts []s3types.CompletedPart for i := 0; i < len(sources); i++ { partNumber := i + 1 - parts = append(parts, &s3.CompletedPart{ + parts = append(parts, s3types.CompletedPart{ ETag: etags[partNumber], - PartNumber: aws.Int64(int64(partNumber)), + PartNumber: int32(partNumber), }) } @@ -250,7 +250,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st Bucket: multipartUpload.Bucket, Key: multipartUpload.Key, UploadId: multipartUpload.UploadId, - MultipartUpload: &s3.CompletedMultipartUpload{Parts: parts}, + MultipartUpload: &s3types.CompletedMultipartUpload{Parts: parts}, }); err != nil { return 0, errors.Wrap(err, "failed to complete multipart upload") } @@ -263,7 +263,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st return 0, errors.Wrap(err, "failed to stat composed object") } - return *obj.ContentLength, nil + return obj.ContentLength, nil } func (s *s3Store) Delete(ctx context.Context, key string) (err error) { @@ -285,15 +285,13 @@ func (s *s3Store) create(ctx context.Context) error { Bucket: aws.String(s.bucket), }) - codes := []string{ - s3.ErrCodeBucketAlreadyExists, - s3.ErrCodeBucketAlreadyOwnedByYou, + var bae *s3types.BucketAlreadyExists + if errors.As(err, &bae) { + return nil } - - for _, code := range codes { - if aerr, ok := err.(awserr.Error); ok && aerr.Code() == code { - return nil - } + var baoby *s3types.BucketAlreadyOwnedByYou + if errors.As(err, &baoby) { + return nil } return err @@ -302,34 +300,13 @@ func (s *s3Store) create(ctx context.Context) error { func (s *s3Store) update(ctx context.Context) error { configureRequest := &s3.PutBucketLifecycleConfigurationInput{ Bucket: aws.String(s.bucket), - LifecycleConfiguration: s.lifecycle(), + LifecycleConfiguration: s.bucketLifecycleConfiguration, } _, err := s.client.PutBucketLifecycleConfiguration(ctx, configureRequest) return err } -func (s *s3Store) lifecycle() *s3.BucketLifecycleConfiguration { - days := aws.Int64(int64(s.ttl / (time.Hour * 24))) - - return &s3.BucketLifecycleConfiguration{ - Rules: []*s3.LifecycleRule{ - { - ID: aws.String("Expiration Rule"), - Status: aws.String("Enabled"), - Filter: &s3.LifecycleRuleFilter{Prefix: aws.String("")}, - Expiration: &s3.LifecycleExpiration{Days: days}, - }, - { - ID: aws.String("Abort Incomplete Multipart Upload Rule"), - Status: aws.String("Enabled"), - Filter: &s3.LifecycleRuleFilter{Prefix: aws.String("")}, - AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{DaysAfterInitiation: days}, - }, - }, - } -} - func (s *s3Store) deleteSources(ctx context.Context, bucket string, sources []string) error { return goroutine.RunWorkersOverStrings(sources, func(index int, source string) error { if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ @@ -356,24 +333,26 @@ func (r *countingReader) Read(p []byte) (n int, err error) { return n, err } -func s3SessionOptions(backend string, config S3Config) session.Options { - creds := credentials.NewStaticCredentials( - config.AccessKeyID, - config.SecretAccessKey, - config.SessionToken, - ) - - awsConfig := aws.Config{ - Credentials: creds, - Region: aws.String(config.Region), +func s3ClientConfig(ctx context.Context, s3config S3Config) (aws.Config, error) { + optFns := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(s3config.Region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + s3config.AccessKeyID, + s3config.SecretAccessKey, + s3config.SessionToken, + )), } - if backend == "minio" { - awsConfig.Endpoint = aws.String(config.Endpoint) - awsConfig.S3ForcePathStyle = aws.Bool(true) - } + return awsconfig.LoadDefaultConfig(ctx, optFns...) +} - return session.Options{Config: awsConfig} +func s3ClientOptions(backend string, config S3Config) func(o *s3.Options) { + return func(o *s3.Options) { + if backend == "minio" { + o.EndpointResolver = s3.EndpointResolverFromURL(config.Endpoint) + o.UsePathStyle = true + } + } } // writeToPipe invokes the given function with a pipe writer in a goroutine @@ -391,3 +370,27 @@ func isConnectionResetError(err error) bool { return false } + +func s3BucketLifecycleConfiguration(backend string, ttl time.Duration) *s3types.BucketLifecycleConfiguration { + days := int32(ttl / (time.Hour * 24)) + + rules := []s3types.LifecycleRule{ + { + ID: aws.String("Expiration Rule"), + Status: s3types.ExpirationStatusEnabled, + Filter: &s3types.LifecycleRuleFilterMemberPrefix{Value: ""}, + Expiration: &s3types.LifecycleExpiration{Days: days}, + }, + } + + if backend != "minio" { + rules = append(rules, s3types.LifecycleRule{ + ID: aws.String("Abort Incomplete Multipart Upload Rule"), + Status: s3types.ExpirationStatusEnabled, + Filter: &s3types.LifecycleRuleFilterMemberPrefix{Value: ""}, + AbortIncompleteMultipartUpload: &s3types.AbortIncompleteMultipartUpload{DaysAfterInitiation: days}, + }) + } + + return &s3types.BucketLifecycleConfiguration{Rules: rules} +} diff --git a/enterprise/internal/codeintel/stores/uploadstore/s3_client_test.go b/enterprise/internal/codeintel/stores/uploadstore/s3_client_test.go index 20dd2846269..2f0760f8525 100644 --- a/enterprise/internal/codeintel/stores/uploadstore/s3_client_test.go +++ b/enterprise/internal/codeintel/stores/uploadstore/s3_client_test.go @@ -12,10 +12,9 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - s3 "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + s3 "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" @@ -43,9 +42,9 @@ func TestS3Init(t *testing.T) { } func TestS3InitBucketExists(t *testing.T) { - for _, code := range []string{s3.ErrCodeBucketAlreadyExists, s3.ErrCodeBucketAlreadyOwnedByYou} { + for _, err := range []error{&s3types.BucketAlreadyExists{}, &s3types.BucketAlreadyOwnedByYou{}} { s3Client := NewMockS3API() - s3Client.CreateBucketFunc.SetDefaultReturn(nil, awserr.New(code, "", nil)) + s3Client.CreateBucketFunc.SetDefaultReturn(nil, err) client := testS3Client(s3Client, nil) if err := client.Init(context.Background()); err != nil { @@ -68,7 +67,7 @@ func TestS3InitBucketExists(t *testing.T) { func TestS3UnmanagedInit(t *testing.T) { s3Client := NewMockS3API() - client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext)) + client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext)) if err := client.Init(context.Background()); err != nil { t.Fatalf("unexpected error initializing client: %s", err) } @@ -87,7 +86,7 @@ func TestS3Get(t *testing.T) { Body: ioutil.NopCloser(bytes.NewReader([]byte("TEST PAYLOAD"))), }, nil) - client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext)) + client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext)) rc, err := client.Get(context.Background(), "test-key") if err != nil { t.Fatalf("unexpected error getting key: %s", err) @@ -139,7 +138,7 @@ func TestS3GetTransientErrors(t *testing.T) { } s3Client := fullContentsS3API() - client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext)) + client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext)) rc, err := client.Get(context.Background(), "test-key") if err != nil { t.Fatalf("unexpected error getting key: %s", err) @@ -168,7 +167,7 @@ func TestS3GetReadNothingLoop(t *testing.T) { } s3Client := fullContentsS3API() - client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext)) + client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext)) rc, err := client.Get(context.Background(), "test-key") if err != nil { t.Fatalf("unexpected error getting key: %s", err) @@ -213,7 +212,7 @@ func fullContentsS3API() *MockS3API { func TestS3Upload(t *testing.T) { s3Client := NewMockS3API() uploaderClient := NewMockS3Uploader() - uploaderClient.UploadFunc.SetDefaultHook(func(ctx context.Context, input *s3manager.UploadInput) error { + uploaderClient.UploadFunc.SetDefaultHook(func(ctx context.Context, input *s3.PutObjectInput) error { // Synchronously read the reader so that we trigger the // counting reader inside the Upload method and test the // count. @@ -257,13 +256,13 @@ func TestS3Combine(t *testing.T) { s3Client.UploadPartCopyFunc.SetDefaultHook(func(ctx context.Context, input *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) { return &s3.UploadPartCopyOutput{ - CopyPartResult: &s3.CopyPartResult{ + CopyPartResult: &s3types.CopyPartResult{ ETag: aws.String(fmt.Sprintf("etag-%s", *input.CopySource)), }, }, nil }) - s3Client.HeadObjectFunc.SetDefaultReturn(&s3.HeadObjectOutput{ContentLength: aws.Int64(42)}, nil) + s3Client.HeadObjectFunc.SetDefaultReturn(&s3.HeadObjectOutput{ContentLength: int64(42)}, nil) client := testS3Client(s3Client, nil) @@ -277,7 +276,7 @@ func TestS3Combine(t *testing.T) { if calls := s3Client.UploadPartCopyFunc.History(); len(calls) != 3 { t.Fatalf("unexpected number of UploadPartCopy calls. want=%d have=%d", 3, len(calls)) } else { - parts := map[int64]string{} + parts := map[int32]string{} for _, call := range calls { if value := *call.Arg1.Bucket; value != "test-bucket" { t.Errorf("unexpected bucket argument. want=%s have=%s", "test-bucket", value) @@ -289,10 +288,10 @@ func TestS3Combine(t *testing.T) { t.Errorf("unexpected key argument. want=%s have=%s", "uid", value) } - parts[*call.Arg1.PartNumber] = *call.Arg1.CopySource + parts[call.Arg1.PartNumber] = *call.Arg1.CopySource } - expectedParts := map[int64]string{ + expectedParts := map[int32]string{ 1: "test-bucket/test-src1", 2: "test-bucket/test-src2", 3: "test-bucket/test-src3", @@ -319,12 +318,12 @@ func TestS3Combine(t *testing.T) { } else if value := *calls[0].Arg1.UploadId; value != "uid" { t.Errorf("unexpected uploadId argument. want=%s have=%s", "uid", value) } else { - parts := map[int64]string{} + parts := map[int32]string{} for _, part := range calls[0].Arg1.MultipartUpload.Parts { - parts[*part.PartNumber] = *part.ETag + parts[part.PartNumber] = *part.ETag } - expectedParts := map[int64]string{ + expectedParts := map[int32]string{ 1: "etag-test-bucket/test-src1", 2: "etag-test-bucket/test-src2", 3: "etag-test-bucket/test-src3", @@ -381,39 +380,28 @@ func TestS3Delete(t *testing.T) { } } -func TestS3Lifecycle(t *testing.T) { - s3Client := NewMockS3API() - client := rawS3Client(s3Client, nil) - - if lifecycle := client.lifecycle(); lifecycle == nil || len(lifecycle.Rules) != 2 { +func TestS3BucketLifecycleConfiguration(t *testing.T) { + if lifecycle := s3BucketLifecycleConfiguration("s3", time.Hour*24*3); lifecycle == nil || len(lifecycle.Rules) != 2 { t.Fatalf("unexpected lifecycle rules") } else { - var objectExpiration *int64 + var objectExpiration int32 for _, rule := range lifecycle.Rules { if rule.Expiration != nil { - if value := rule.Expiration.Days; value != nil { - objectExpiration = value - } + objectExpiration = rule.Expiration.Days } } - if objectExpiration == nil { - t.Fatalf("expected object expiration to be configured") - } else if *objectExpiration != 3 { - t.Errorf("unexpected ttl for object expiration. want=%d have=%d", 3, *objectExpiration) + if objectExpiration != 3 { + t.Errorf("unexpected ttl for object expiration. want=%d have=%d", 3, objectExpiration) } - var multipartExpiration *int64 + var multipartExpiration int32 for _, rule := range lifecycle.Rules { if rule.AbortIncompleteMultipartUpload != nil { - if value := rule.AbortIncompleteMultipartUpload.DaysAfterInitiation; value != nil { - multipartExpiration = value - } + multipartExpiration = rule.AbortIncompleteMultipartUpload.DaysAfterInitiation } } - if multipartExpiration == nil { - t.Fatalf("expected multipart upload expiration to be configured") - } else if *multipartExpiration != 3 { - t.Errorf("unexpected ttl for multipart upload expiration. want=%d have=%d", 3, *multipartExpiration) + if multipartExpiration != 3 { + t.Errorf("unexpected ttl for multipart upload expiration. want=%d have=%d", 3, multipartExpiration) } } } @@ -423,5 +411,5 @@ func testS3Client(client s3API, uploader s3Uploader) Store { } func rawS3Client(client s3API, uploader s3Uploader) *s3Store { - return newS3WithClients(client, uploader, "test-bucket", time.Hour*24*3, true, newOperations(&observation.TestContext)) + return newS3WithClients(client, uploader, "test-bucket", true, nil, newOperations(&observation.TestContext)) } diff --git a/go.mod b/go.mod index f8f6015ef38..8712e0991f5 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,13 @@ require ( github.com/aphistic/sweet-junit v0.2.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/avelino/slugify v0.0.0-20180501145920-855f152bd774 - github.com/aws/aws-sdk-go v1.29.15 - github.com/aws/aws-sdk-go-v2 v0.20.0 + github.com/aws/aws-sdk-go-v2 v1.2.1 + github.com/aws/aws-sdk-go-v2/config v1.1.2 + github.com/aws/aws-sdk-go-v2/credentials v1.1.2 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3 + github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1 + github.com/aws/smithy-go v1.2.0 github.com/beevik/etree v1.1.0 github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect @@ -96,7 +101,6 @@ require ( github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1 github.com/jackc/pgconn v1.8.0 github.com/jackc/pgx/v4 v4.10.0 - github.com/jmespath/go-jmespath v0.3.0 // indirect github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 github.com/joho/godotenv v1.3.0 github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible diff --git a/go.sum b/go.sum index 5de2a3e5429..d28ba8cddd4 100644 --- a/go.sum +++ b/go.sum @@ -138,11 +138,34 @@ github.com/avelino/slugify v0.0.0-20180501145920-855f152bd774/go.mod h1:5wi5YYOp github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs= github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/aws/aws-sdk-go-v2 v0.20.0 h1:/yefUjgMrda9PNFwWctBU63nL10CJMdBwkAmaQ4w4Hs= -github.com/aws/aws-sdk-go-v2 v0.20.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw= +github.com/aws/aws-sdk-go-v2 v1.2.1 h1:055XAi+MtmhyYX161p+jWRibkCb9YpI2ymXZiW1dwVY= +github.com/aws/aws-sdk-go-v2 v1.2.1/go.mod h1:hTQc/9pYq5bfFACIUY9tc/2SYWd9Vnmw+testmuQeRY= +github.com/aws/aws-sdk-go-v2/config v1.1.2 h1:H2r6cwMvvINFpEC55Y7jcNaR/oc7zYIChrG2497wmBI= +github.com/aws/aws-sdk-go-v2/config v1.1.2/go.mod h1:77yIk+qmCS/94JlxbwV1d+YEyu6Z8FBlCGcSz3TdM6A= +github.com/aws/aws-sdk-go-v2/credentials v1.1.2 h1:YoNqfhxAJGZI+lStIbqgx30UcCqQ86fr7FjTLUvrFOc= +github.com/aws/aws-sdk-go-v2/credentials v1.1.2/go.mod h1:hofjw//lM0XLplgvzPPMA7oD0doQU1QpaIK1nweEEWg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3 h1:d3bKAGy4XdJyK8hz3Nx3WJJ4TCmYp2498G4mFY5wly0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3/go.mod h1:Zr1Mj+KUMGVQ+WJvTT68EZJxqhjiie2PWSPGEUPaNY0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3 h1:vhRq0752KGBMmLnVessDOpt+5XEdzM87hhiuwGiEpqc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3/go.mod h1:9tFvXNMet5TrBa2bMLhZBvenXs4qKMqiG1n0MNR4FFA= +github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2 h1:PIAYXYIBEBFaSvCNTiXsxakePOXC2eZ+wpxWhD2J4O4= +github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2/go.mod h1:9Y2XYeoa34KWUK9wae2Iuel8OLRxc00SGgIlGUCOoXE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2 h1:GO0pL4QvQmA0fXJe3MHVO+emtg31MYq5/8sebSWgE6A= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2/go.mod h1:bYl7lGFQQdHia3uMQH4p6ImnuOeDNeUoydoXM5x8Yzw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3 h1:dST4y8pZKZdTPs4uwXmGCJmpycz1SHKmCSIhf3GqHEo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3/go.mod h1:C50Z41fJaJ7WgaeeCulOGAU3q4+4se4B3uOPFdhBi2I= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1 h1:+WCVceRPiUsrui55mDByXOVremK1n3Hm8GnB4ZD3eco= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1/go.mod h1:B+fb+BFbja6obFOHYmYE4iUMdej9aM2VGSpgdU1pn0M= +github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1 h1:3qn6YVXpOCK9seQ8ZilDyMrhpEUaZNaJG8SXNiCvk+c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1/go.mod h1:3xGOyhtPPD/WXJUljmb5+ZXhNyHa4h6wgL6mWOF6S0c= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.2 h1:9BnjX/ALn5uLo2DbgkwMpUkPL1VLQVBXcjZxqJBhf44= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.2/go.mod h1:5yU1oE3+CVYYLUsaHt2AVU3CJJZ6ER4pwsrRD1L2KSc= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.2 h1:7Kxqov7uQeP8WUEO0iHz3j9Bh0E1rJrn6cf/OGfcDds= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.2/go.mod h1:zu7rotIY9P4Aoc6ytqLP9jeYrECDHUODB5Gbp+BSHl8= +github.com/aws/smithy-go v1.2.0 h1:0PoGBWXkXDIyVdPaZW9gMhaGzj3UOAgTdiVoHuuZAFA= +github.com/aws/smithy-go v1.2.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= @@ -831,8 +854,10 @@ github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:x github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 h1:lrdPtrORjGv1HbbEvKWDUAy97mPpFm4B8hp77tcCUJY= github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= diff --git a/internal/extsvc/awscodecommit/client.go b/internal/extsvc/awscodecommit/client.go index 71c9978e7e4..52726776ad6 100644 --- a/internal/extsvc/awscodecommit/client.go +++ b/internal/extsvc/awscodecommit/client.go @@ -6,8 +6,8 @@ import ( "encoding/hex" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/awserr" - "github.com/aws/aws-sdk-go-v2/service/codecommit" + codecommittypes "github.com/aws/aws-sdk-go-v2/service/codecommit/types" + "github.com/aws/smithy-go" "github.com/pkg/errors" "github.com/sourcegraph/sourcegraph/internal/rcache" @@ -53,16 +53,15 @@ func IsNotFound(err error) bool { if err == ErrNotFound || errors.Cause(err) == ErrNotFound { return true } - if e, ok := err.(awserr.Error); ok { - return e.Code() == codecommit.ErrCodeRepositoryDoesNotExistException - } - return false + var rdnee codecommittypes.RepositoryDoesNotExistException + return errors.As(err, &rdnee) } // IsUnauthorized reports whether err is a AWS CodeCommit API unauthorized error. func IsUnauthorized(err error) bool { - if e, ok := err.(awserr.Error); ok { - return e.Code() == "SignatureDoesNotMatch" + var ae smithy.APIError + if errors.As(err, &ae) { + return ae.ErrorCode() == "SignatureDoesNotMatch" } return false } diff --git a/internal/extsvc/awscodecommit/codehost.go b/internal/extsvc/awscodecommit/codehost.go index 3033322b851..94c624059d5 100644 --- a/internal/extsvc/awscodecommit/codehost.go +++ b/internal/extsvc/awscodecommit/codehost.go @@ -1,8 +1,6 @@ package awscodecommit import ( - "github.com/aws/aws-sdk-go-v2/aws/endpoints" - "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/extsvc" ) @@ -22,6 +20,6 @@ func ExternalRepoSpec(repo *Repository, serviceID string) api.ExternalRepoSpec { // // This value uniquely identifies the most specific namespace in which AWS CodeCommit repositories // are defined. -func ServiceID(awsPartition endpoints.Partition, awsRegion endpoints.Region, awsAccountID string) string { - return "arn:" + awsPartition.ID() + ":codecommit:" + awsRegion.ID() + ":" + awsAccountID + ":" +func ServiceID(awsPartition, awsRegion, awsAccountID string) string { + return "arn:" + awsPartition + ":codecommit:" + awsRegion + ":" + awsAccountID + ":" } diff --git a/internal/extsvc/awscodecommit/repos.go b/internal/extsvc/awscodecommit/repos.go index 303211ea1a7..5db89a2c0f7 100644 --- a/internal/extsvc/awscodecommit/repos.go +++ b/internal/extsvc/awscodecommit/repos.go @@ -7,6 +7,7 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/codecommit" + codecommittypes "github.com/aws/aws-sdk-go-v2/service/codecommit/types" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -132,10 +133,8 @@ func (c *Client) getRepositoryFromAPI(ctx context.Context, arn string) (*Reposit repoName = arn[i+1:] } - svc := codecommit.New(c.aws) - req := svc.GetRepositoryRequest(&codecommit.GetRepositoryInput{RepositoryName: &repoName}) - req.SetContext(ctx) - result, err := req.Send(ctx) + svc := codecommit.NewFromConfig(c.aws) + result, err := svc.GetRepository(ctx, &codecommit.GetRepositoryInput{RepositoryName: &repoName}) if err != nil { return nil, err } @@ -154,19 +153,17 @@ func (c *Client) ListRepositories(ctx context.Context, nextToken string) (repos } }() - svc := codecommit.New(c.aws) + svc := codecommit.NewFromConfig(c.aws) // List repositories. listInput := codecommit.ListRepositoriesInput{ - Order: codecommit.OrderEnumDescending, - SortBy: codecommit.SortByEnumLastModifiedDate, + Order: codecommittypes.OrderEnumDescending, + SortBy: codecommittypes.SortByEnumModifiedDate, } if nextToken != "" { listInput.NextToken = &nextToken } - listReq := svc.ListRepositoriesRequest(&listInput) - listReq.SetContext(ctx) - listResult, err := listReq.Send(ctx) + listResult, err := svc.ListRepositories(ctx, &listInput) if err != nil { return nil, "", err } @@ -201,9 +198,7 @@ func (c *Client) ListRepositories(ctx context.Context, nextToken string) (repos func (c *Client) getRepositories(ctx context.Context, svc *codecommit.Client, repositoryNames []string) ([]*Repository, error) { getInput := codecommit.BatchGetRepositoriesInput{RepositoryNames: repositoryNames} - getReq := svc.BatchGetRepositoriesRequest(&getInput) - getReq.SetContext(ctx) - getResult, err := getReq.Send(ctx) + getResult, err := svc.BatchGetRepositories(ctx, &getInput) if err != nil { return nil, err } @@ -243,7 +238,7 @@ func (w *wrappedError) Unauthorized() bool { return IsUnauthorized(w.err) } -func fromRepoMetadata(m *codecommit.RepositoryMetadata) *Repository { +func fromRepoMetadata(m *codecommittypes.RepositoryMetadata) *Repository { repo := Repository{ ARN: *m.Arn, AccountID: *m.AccountId, diff --git a/internal/repos/awscodecommit.go b/internal/repos/awscodecommit.go index 02fc85469a0..51b6c1eccd7 100644 --- a/internal/repos/awscodecommit.go +++ b/internal/repos/awscodecommit.go @@ -7,9 +7,12 @@ import ( "net/url" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/defaults" - "github.com/aws/aws-sdk-go-v2/aws/endpoints" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" + awscredentials "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/codecommit" "github.com/inconshreveable/log15" + "github.com/pkg/errors" "golang.org/x/net/http2" "github.com/sourcegraph/sourcegraph/internal/conf/reposource" @@ -27,9 +30,8 @@ type AWSCodeCommitSource struct { svc *types.ExternalService config *schema.AWSCodeCommitConnection - awsConfig aws.Config - awsPartition endpoints.Partition // "aws", "aws-cn", "aws-us-gov" - awsRegion endpoints.Region + awsPartition string // "aws", "aws-cn", "aws-us-gov" + awsRegion string client *awscodecommit.Client exclude excludeFunc @@ -45,22 +47,12 @@ func NewAWSCodeCommitSource(svc *types.ExternalService, cf *httpcli.Factory) (*A } func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitConnection, cf *httpcli.Factory) (*AWSCodeCommitSource, error) { - awsConfig := defaults.Config() - awsConfig.Region = c.Region - awsConfig.Credentials = aws.StaticCredentialsProvider{ - Value: aws.Credentials{ - AccessKeyID: c.AccessKeyID, - SecretAccessKey: c.SecretAccessKey, - Source: "sourcegraph-site-configuration", - }, - } - if cf == nil { cf = httpcli.NewExternalHTTPClientFactory() } cli, err := cf.Doer(func(c *http.Client) error { - tr := aws.NewBuildableHTTPClient().GetTransport() + tr := awshttp.NewBuildableClient().GetTransport() if err := http2.ConfigureTransport(tr); err != nil { return err } @@ -72,7 +64,23 @@ func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitC if err != nil { return nil, err } - awsConfig.HTTPClient = cli + + awsConfig, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(c.Region), + config.WithCredentialsProvider( + awscredentials.StaticCredentialsProvider{ + Value: aws.Credentials{ + AccessKeyID: c.AccessKeyID, + SecretAccessKey: c.SecretAccessKey, + Source: "sourcegraph-site-configuration", + }, + }, + ), + config.WithHTTPClient(cli), + ) + if err != nil { + return nil, err + } var eb excludeBuilder for _, r := range c.Exclude { @@ -85,21 +93,18 @@ func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitC } s := &AWSCodeCommitSource{ - svc: svc, - config: c, - awsConfig: awsConfig, - exclude: exclude, - client: awscodecommit.NewClient(awsConfig), + svc: svc, + config: c, + exclude: exclude, + client: awscodecommit.NewClient(awsConfig), } - var ok bool - s.awsPartition, ok = endpoints.DefaultPartitions().ForRegion(c.Region) - if ok { - s.awsRegion, ok = s.awsPartition.Regions()[c.Region] - } - if !ok { - return nil, fmt.Errorf("unrecognized AWS region name: %q", c.Region) + endpoint, err := codecommit.NewDefaultEndpointResolver().ResolveEndpoint(c.Region, codecommit.EndpointResolverOptions{}) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("failed to resolve AWS region %q", c.Region)) } + s.awsPartition = endpoint.PartitionID + s.awsRegion = endpoint.SigningRegion return s, nil }