Upgrade aws-sdk-go-v2 (#19155)

This PR:
- Migrates to the AWS v2 API in the codeintel s3 adapter
- Upgrades the v2 package to the newest version
This commit is contained in:
Erik Seliger 2021-04-14 15:06:15 +02:00 committed by GitHub
parent 28c81089e8
commit 2a3bd2b515
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 289 additions and 242 deletions

View File

@ -54,7 +54,7 @@ COPY --from=comby/comby:0.18.4@sha256:b47ce282778bfea7f80d45f5ef0cc546ba0d6347ba
COPY --from=sourcegraph/syntect_server:331beda@sha256:6b8950b41993af3d10300b5a160fab1c06c337cb4614978f4fdb76e2588afbfe /syntect_server /usr/local/bin/
# install minio (keep this up to date with docker-images/minio/Dockerfile)
ENV MINIO_VERSION=RELEASE.2020-10-18T21-54-12Z
ENV MINIO_VERSION=RELEASE.2021-04-06T23-11-00Z
RUN wget "https://dl.min.io/server/minio/release/linux-amd64/archive/minio.$MINIO_VERSION" && \
chmod +x "minio.$MINIO_VERSION" && \
mv "minio.$MINIO_VERSION" /usr/local/bin/minio

View File

@ -14,9 +14,6 @@ go get -u all
## names. This is causing our use of our k8s client to panic.
go get github.com/golang/protobuf@v1.3.5
## Newer versions removed some types in the endpoint package we relied on. Unsure how to fix yet, so punting.
go get github.com/aws/aws-sdk-go-v2@v0.20.0
# Cleanup and validate everything still works
go mod tidy
go test -short -failfast ./... >/dev/null

View File

@ -346,3 +346,10 @@
:why: Internal module
:versions: []
:when: 2021-03-09 19:42:12.214862934 Z
- - :license
- github.com/aws/smithy-go
- Apache 2.0
- :who:
:why: Inference broken, LICENSE file lives at https://github.com/aws/smithy-go/blob/main/LICENSE
:versions: []
:when: 2021-04-14 12:51:19.503802000 Z

View File

@ -3,7 +3,7 @@
set -ex
cd "$(dirname "${BASH_SOURCE[0]}")"
# Retag the 2020-10-18 MinIO release
MINIO_RELEASE="RELEASE.2020-10-18T21-54-12Z"
# Retag the 2021-04-06 MinIO release
MINIO_RELEASE="RELEASE.2021-04-06T23-11-00Z"
docker pull minio/minio:$MINIO_RELEASE
docker tag minio/minio:$MINIO_RELEASE "$IMAGE"

View File

@ -8,7 +8,7 @@ import (
"net/http"
"time"
"github.com/aws/aws-sdk-go/aws/awserr"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/inconshreveable/log15"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
@ -199,15 +199,6 @@ func initializeUploadStore(ctx context.Context, uploadStore uploadstore.Store) e
}
func isRequestError(err error) bool {
for err != nil {
if e, ok := err.(awserr.Error); ok {
if e.Code() == "RequestError" {
return true
}
}
err = errors.Unwrap(err)
}
return false
var rse *smithyhttp.RequestSendError
return errors.As(err, &rse)
}

View File

@ -1,8 +1,13 @@
package uploadstore
import (
"context"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/go-cmp/cmp"
)
func TestConfigDefaults(t *testing.T) {
@ -70,25 +75,44 @@ func TestConfigS3(t *testing.T) {
}
}
func TestConfigMinIOSessionOptions(t *testing.T) {
func TestS3ClientOptions(t *testing.T) {
config := Config{}
config.SetMockGetter(mapGetter(nil))
config.Load()
options := s3SessionOptions("minio", config.S3)
// minIO
{
options := &s3.Options{}
s3ClientOptions("minio", config.S3)(options)
if value := *options.Config.Region; value != "us-east-1" {
t.Errorf("unexpected region. want=%s have=%s", "us-east-1", value)
if options.EndpointResolver == nil {
t.Fatalf("unexpected endpoint option")
}
endpoint, err := options.EndpointResolver.ResolveEndpoint("us-east-2", s3.EndpointResolverOptions{})
if err != nil {
t.Fatal(err)
}
if endpoint.URL != "http://minio:9000" {
t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", endpoint.URL)
}
if !options.UsePathStyle {
t.Errorf("invalid UsePathStyle setting for S3Options")
}
}
if value := *options.Config.Endpoint; value != "http://minio:9000" {
t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", value)
}
if options.Config.S3ForcePathStyle == nil || !*options.Config.S3ForcePathStyle {
t.Errorf("expected path style option")
// S3
{
options := &s3.Options{}
s3ClientOptions("s3", config.S3)(options)
if diff := cmp.Diff(&s3.Options{}, options); diff != "" {
t.Fatalf("invalid s3 options returned for S3: %s", diff)
}
}
}
func TestConfigS3SessionOptions(t *testing.T) {
func TestS3ClientConfig(t *testing.T) {
env := map[string]string{
"PRECISE_CODE_INTEL_UPLOAD_BACKEND": "S3",
"PRECISE_CODE_INTEL_UPLOAD_BUCKET": "lsif-uploads",
@ -104,16 +128,28 @@ func TestConfigS3SessionOptions(t *testing.T) {
config.SetMockGetter(mapGetter(env))
config.Load()
options := s3SessionOptions("s3", config.S3)
cfg, err := s3ClientConfig(context.Background(), config.S3)
if err != nil {
t.Fatal(err)
}
if value := *options.Config.Region; value != "us-east-2" {
if value := cfg.Region; value != "us-east-2" {
t.Errorf("unexpected region. want=%s have=%s", "us-east-2", value)
}
if options.Config.Endpoint != nil {
t.Errorf("unexpected endpoint option")
cred, err := cfg.Credentials.Retrieve(context.Background())
if err != nil {
t.Fatal(err)
}
if options.Config.S3ForcePathStyle != nil {
t.Errorf("unexpected path style option")
if diff := cmp.Diff(aws.Credentials{
AccessKeyID: config.S3.AccessKeyID,
SecretAccessKey: config.S3.SecretAccessKey,
SessionToken: config.S3.SessionToken,
Source: "StaticCredentials",
}, cred); diff != "" {
t.Errorf("invalid credential returned: %s", diff)
}
if cfg.EndpointResolver != nil {
t.Errorf("unexpected endpoint option")
}
}

View File

@ -6,8 +6,7 @@ import (
"context"
"sync"
s3 "github.com/aws/aws-sdk-go/service/s3"
s3manager "github.com/aws/aws-sdk-go/service/s3/s3manager"
s3 "github.com/aws/aws-sdk-go-v2/service/s3"
)
// MockS3API is a mock implementation of the s3API interface (from the
@ -1143,7 +1142,7 @@ type MockS3Uploader struct {
func NewMockS3Uploader() *MockS3Uploader {
return &MockS3Uploader{
UploadFunc: &S3UploaderUploadFunc{
defaultHook: func(context.Context, *s3manager.UploadInput) error {
defaultHook: func(context.Context, *s3.PutObjectInput) error {
return nil
},
},
@ -1155,7 +1154,7 @@ func NewMockS3Uploader() *MockS3Uploader {
// github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/uploadstore).
// It is redefined here as it is unexported in the source packge.
type surrogateMockS3Uploader interface {
Upload(context.Context, *s3manager.UploadInput) error
Upload(context.Context, *s3.PutObjectInput) error
}
// NewMockS3UploaderFrom creates a new mock of the MockS3Uploader interface.
@ -1171,15 +1170,15 @@ func NewMockS3UploaderFrom(i surrogateMockS3Uploader) *MockS3Uploader {
// S3UploaderUploadFunc describes the behavior when the Upload method of the
// parent MockS3Uploader instance is invoked.
type S3UploaderUploadFunc struct {
defaultHook func(context.Context, *s3manager.UploadInput) error
hooks []func(context.Context, *s3manager.UploadInput) error
defaultHook func(context.Context, *s3.PutObjectInput) error
hooks []func(context.Context, *s3.PutObjectInput) error
history []S3UploaderUploadFuncCall
mutex sync.Mutex
}
// Upload delegates to the next hook function in the queue and stores the
// parameter and result values of this invocation.
func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3manager.UploadInput) error {
func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3.PutObjectInput) error {
r0 := m.UploadFunc.nextHook()(v0, v1)
m.UploadFunc.appendCall(S3UploaderUploadFuncCall{v0, v1, r0})
return r0
@ -1187,7 +1186,7 @@ func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3manager.UploadInput) e
// SetDefaultHook sets function that is called when the Upload method of the
// parent MockS3Uploader instance is invoked and the hook queue is empty.
func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3manager.UploadInput) error) {
func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3.PutObjectInput) error) {
f.defaultHook = hook
}
@ -1195,7 +1194,7 @@ func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3mana
// Upload method of the parent MockS3Uploader instance invokes the hook at
// the front of the queue and discards it. After the queue is empty, the
// default hook function is invoked for any future action.
func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3manager.UploadInput) error) {
func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3.PutObjectInput) error) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
@ -1204,7 +1203,7 @@ func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3manager.Up
// SetDefaultReturn calls SetDefaultDefaultHook with a function that returns
// the given values.
func (f *S3UploaderUploadFunc) SetDefaultReturn(r0 error) {
f.SetDefaultHook(func(context.Context, *s3manager.UploadInput) error {
f.SetDefaultHook(func(context.Context, *s3.PutObjectInput) error {
return r0
})
}
@ -1212,12 +1211,12 @@ func (f *S3UploaderUploadFunc) SetDefaultReturn(r0 error) {
// PushReturn calls PushDefaultHook with a function that returns the given
// values.
func (f *S3UploaderUploadFunc) PushReturn(r0 error) {
f.PushHook(func(context.Context, *s3manager.UploadInput) error {
f.PushHook(func(context.Context, *s3.PutObjectInput) error {
return r0
})
}
func (f *S3UploaderUploadFunc) nextHook() func(context.Context, *s3manager.UploadInput) error {
func (f *S3UploaderUploadFunc) nextHook() func(context.Context, *s3.PutObjectInput) error {
f.mutex.Lock()
defer f.mutex.Unlock()
@ -1255,7 +1254,7 @@ type S3UploaderUploadFuncCall struct {
Arg0 context.Context
// Arg1 is the value of the 2nd argument passed to this method
// invocation.
Arg1 *s3manager.UploadInput
Arg1 *s3.PutObjectInput
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 error

View File

@ -3,8 +3,8 @@ package uploadstore
import (
"context"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type s3API interface {
@ -20,52 +20,52 @@ type s3API interface {
}
type s3Uploader interface {
Upload(ctx context.Context, input *s3manager.UploadInput) error
Upload(ctx context.Context, input *s3.PutObjectInput) error
}
type s3APIShim struct{ *s3.S3 }
type s3UploaderShim struct{ *s3manager.Uploader }
type s3APIShim struct{ *s3.Client }
type s3UploaderShim struct{ *manager.Uploader }
var _ s3API = &s3APIShim{}
var _ s3Uploader = &s3UploaderShim{}
func (s *s3APIShim) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) (*s3.CreateBucketOutput, error) {
return s.S3.CreateBucketWithContext(ctx, input)
return s.Client.CreateBucket(ctx, input)
}
func (s *s3APIShim) PutBucketLifecycleConfiguration(ctx context.Context, input *s3.PutBucketLifecycleConfigurationInput) (*s3.PutBucketLifecycleConfigurationOutput, error) {
return s.S3.PutBucketLifecycleConfigurationWithContext(ctx, input)
return s.Client.PutBucketLifecycleConfiguration(ctx, input)
}
func (s *s3APIShim) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return s.S3.HeadObjectWithContext(ctx, input)
return s.Client.HeadObject(ctx, input)
}
func (s *s3APIShim) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
return s.S3.GetObjectWithContext(ctx, input)
return s.Client.GetObject(ctx, input)
}
func (s *s3APIShim) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return s.S3.DeleteObjectWithContext(ctx, input)
return s.Client.DeleteObject(ctx, input)
}
func (s *s3APIShim) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return s.S3.CreateMultipartUploadWithContext(ctx, input)
return s.Client.CreateMultipartUpload(ctx, input)
}
func (s *s3APIShim) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) {
return s.S3.AbortMultipartUploadWithContext(ctx, input)
return s.Client.AbortMultipartUpload(ctx, input)
}
func (s *s3APIShim) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
return s.S3.UploadPartCopyWithContext(ctx, input)
return s.Client.UploadPartCopy(ctx, input)
}
func (s *s3APIShim) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return s.S3.CompleteMultipartUploadWithContext(ctx, input)
return s.Client.CompleteMultipartUpload(ctx, input)
}
func (s *s3UploaderShim) Upload(ctx context.Context, input *s3manager.UploadInput) error {
_, err := s.Uploader.UploadWithContext(ctx, input)
func (s *s3UploaderShim) Upload(ctx context.Context, input *s3.PutObjectInput) error {
_, err := s.Uploader.Upload(ctx, input)
return err
}

View File

@ -9,12 +9,12 @@ import (
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/inconshreveable/log15"
"github.com/opentracing/opentracing-go/log"
"github.com/pkg/errors"
@ -25,12 +25,12 @@ import (
)
type s3Store struct {
bucket string
ttl time.Duration
manageBucket bool
client s3API
uploader s3Uploader
operations *operations
bucket string
manageBucket bool
client s3API
uploader s3Uploader
bucketLifecycleConfiguration *s3types.BucketLifecycleConfiguration
operations *operations
}
var _ Store = &s3Store{}
@ -53,25 +53,25 @@ func (c *S3Config) load(parent *env.BaseConfig) {
// newS3FromConfig creates a new store backed by AWS Simple Storage Service.
func newS3FromConfig(ctx context.Context, config *Config, operations *operations) (Store, error) {
sess, err := session.NewSessionWithOptions(s3SessionOptions(config.Backend, config.S3))
cfg, err := s3ClientConfig(ctx, config.S3)
if err != nil {
return nil, err
}
s3Client := s3.New(sess)
s3Client := s3.NewFromConfig(cfg, s3ClientOptions(config.Backend, config.S3))
api := &s3APIShim{s3Client}
uploader := &s3UploaderShim{s3manager.NewUploaderWithClient(s3Client)}
return newS3WithClients(api, uploader, config.Bucket, config.TTL, config.ManageBucket, operations), nil
uploader := &s3UploaderShim{manager.NewUploader(s3Client)}
return newS3WithClients(api, uploader, config.Bucket, config.ManageBucket, s3BucketLifecycleConfiguration(config.Backend, config.TTL), operations), nil
}
func newS3WithClients(client s3API, uploader s3Uploader, bucket string, ttl time.Duration, manageBucket bool, operations *operations) *s3Store {
func newS3WithClients(client s3API, uploader s3Uploader, bucket string, manageBucket bool, lifecycleConfiguration *s3types.BucketLifecycleConfiguration, operations *operations) *s3Store {
return &s3Store{
bucket: bucket,
ttl: ttl,
manageBucket: manageBucket,
client: client,
uploader: uploader,
operations: operations,
bucket: bucket,
manageBucket: manageBucket,
client: client,
uploader: uploader,
operations: operations,
bucketLifecycleConfiguration: lifecycleConfiguration,
}
}
@ -166,7 +166,7 @@ func (s *s3Store) Upload(ctx context.Context, key string, r io.Reader) (_ int64,
cr := &countingReader{r: r}
if err := s.uploader.Upload(ctx, &s3manager.UploadInput{
if err := s.uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
Body: cr,
@ -220,7 +220,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
Bucket: multipartUpload.Bucket,
Key: multipartUpload.Key,
UploadId: multipartUpload.UploadId,
PartNumber: aws.Int64(int64(partNumber)),
PartNumber: int32(partNumber),
CopySource: aws.String(fmt.Sprintf("%s/%s", s.bucket, source)),
})
if err != nil {
@ -236,13 +236,13 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
return 0, err
}
var parts []*s3.CompletedPart
var parts []s3types.CompletedPart
for i := 0; i < len(sources); i++ {
partNumber := i + 1
parts = append(parts, &s3.CompletedPart{
parts = append(parts, s3types.CompletedPart{
ETag: etags[partNumber],
PartNumber: aws.Int64(int64(partNumber)),
PartNumber: int32(partNumber),
})
}
@ -250,7 +250,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
Bucket: multipartUpload.Bucket,
Key: multipartUpload.Key,
UploadId: multipartUpload.UploadId,
MultipartUpload: &s3.CompletedMultipartUpload{Parts: parts},
MultipartUpload: &s3types.CompletedMultipartUpload{Parts: parts},
}); err != nil {
return 0, errors.Wrap(err, "failed to complete multipart upload")
}
@ -263,7 +263,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
return 0, errors.Wrap(err, "failed to stat composed object")
}
return *obj.ContentLength, nil
return obj.ContentLength, nil
}
func (s *s3Store) Delete(ctx context.Context, key string) (err error) {
@ -285,15 +285,13 @@ func (s *s3Store) create(ctx context.Context) error {
Bucket: aws.String(s.bucket),
})
codes := []string{
s3.ErrCodeBucketAlreadyExists,
s3.ErrCodeBucketAlreadyOwnedByYou,
var bae *s3types.BucketAlreadyExists
if errors.As(err, &bae) {
return nil
}
for _, code := range codes {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == code {
return nil
}
var baoby *s3types.BucketAlreadyOwnedByYou
if errors.As(err, &baoby) {
return nil
}
return err
@ -302,34 +300,13 @@ func (s *s3Store) create(ctx context.Context) error {
func (s *s3Store) update(ctx context.Context) error {
configureRequest := &s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(s.bucket),
LifecycleConfiguration: s.lifecycle(),
LifecycleConfiguration: s.bucketLifecycleConfiguration,
}
_, err := s.client.PutBucketLifecycleConfiguration(ctx, configureRequest)
return err
}
func (s *s3Store) lifecycle() *s3.BucketLifecycleConfiguration {
days := aws.Int64(int64(s.ttl / (time.Hour * 24)))
return &s3.BucketLifecycleConfiguration{
Rules: []*s3.LifecycleRule{
{
ID: aws.String("Expiration Rule"),
Status: aws.String("Enabled"),
Filter: &s3.LifecycleRuleFilter{Prefix: aws.String("")},
Expiration: &s3.LifecycleExpiration{Days: days},
},
{
ID: aws.String("Abort Incomplete Multipart Upload Rule"),
Status: aws.String("Enabled"),
Filter: &s3.LifecycleRuleFilter{Prefix: aws.String("")},
AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{DaysAfterInitiation: days},
},
},
}
}
func (s *s3Store) deleteSources(ctx context.Context, bucket string, sources []string) error {
return goroutine.RunWorkersOverStrings(sources, func(index int, source string) error {
if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
@ -356,24 +333,26 @@ func (r *countingReader) Read(p []byte) (n int, err error) {
return n, err
}
func s3SessionOptions(backend string, config S3Config) session.Options {
creds := credentials.NewStaticCredentials(
config.AccessKeyID,
config.SecretAccessKey,
config.SessionToken,
)
awsConfig := aws.Config{
Credentials: creds,
Region: aws.String(config.Region),
func s3ClientConfig(ctx context.Context, s3config S3Config) (aws.Config, error) {
optFns := []func(*awsconfig.LoadOptions) error{
awsconfig.WithRegion(s3config.Region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
s3config.AccessKeyID,
s3config.SecretAccessKey,
s3config.SessionToken,
)),
}
if backend == "minio" {
awsConfig.Endpoint = aws.String(config.Endpoint)
awsConfig.S3ForcePathStyle = aws.Bool(true)
}
return awsconfig.LoadDefaultConfig(ctx, optFns...)
}
return session.Options{Config: awsConfig}
func s3ClientOptions(backend string, config S3Config) func(o *s3.Options) {
return func(o *s3.Options) {
if backend == "minio" {
o.EndpointResolver = s3.EndpointResolverFromURL(config.Endpoint)
o.UsePathStyle = true
}
}
}
// writeToPipe invokes the given function with a pipe writer in a goroutine
@ -391,3 +370,27 @@ func isConnectionResetError(err error) bool {
return false
}
func s3BucketLifecycleConfiguration(backend string, ttl time.Duration) *s3types.BucketLifecycleConfiguration {
days := int32(ttl / (time.Hour * 24))
rules := []s3types.LifecycleRule{
{
ID: aws.String("Expiration Rule"),
Status: s3types.ExpirationStatusEnabled,
Filter: &s3types.LifecycleRuleFilterMemberPrefix{Value: ""},
Expiration: &s3types.LifecycleExpiration{Days: days},
},
}
if backend != "minio" {
rules = append(rules, s3types.LifecycleRule{
ID: aws.String("Abort Incomplete Multipart Upload Rule"),
Status: s3types.ExpirationStatusEnabled,
Filter: &s3types.LifecycleRuleFilterMemberPrefix{Value: ""},
AbortIncompleteMultipartUpload: &s3types.AbortIncompleteMultipartUpload{DaysAfterInitiation: days},
})
}
return &s3types.BucketLifecycleConfiguration{Rules: rules}
}

View File

@ -12,10 +12,9 @@ import (
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
s3 "github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
s3 "github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/go-cmp/cmp"
"github.com/pkg/errors"
@ -43,9 +42,9 @@ func TestS3Init(t *testing.T) {
}
func TestS3InitBucketExists(t *testing.T) {
for _, code := range []string{s3.ErrCodeBucketAlreadyExists, s3.ErrCodeBucketAlreadyOwnedByYou} {
for _, err := range []error{&s3types.BucketAlreadyExists{}, &s3types.BucketAlreadyOwnedByYou{}} {
s3Client := NewMockS3API()
s3Client.CreateBucketFunc.SetDefaultReturn(nil, awserr.New(code, "", nil))
s3Client.CreateBucketFunc.SetDefaultReturn(nil, err)
client := testS3Client(s3Client, nil)
if err := client.Init(context.Background()); err != nil {
@ -68,7 +67,7 @@ func TestS3InitBucketExists(t *testing.T) {
func TestS3UnmanagedInit(t *testing.T) {
s3Client := NewMockS3API()
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
if err := client.Init(context.Background()); err != nil {
t.Fatalf("unexpected error initializing client: %s", err)
}
@ -87,7 +86,7 @@ func TestS3Get(t *testing.T) {
Body: ioutil.NopCloser(bytes.NewReader([]byte("TEST PAYLOAD"))),
}, nil)
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
rc, err := client.Get(context.Background(), "test-key")
if err != nil {
t.Fatalf("unexpected error getting key: %s", err)
@ -139,7 +138,7 @@ func TestS3GetTransientErrors(t *testing.T) {
}
s3Client := fullContentsS3API()
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
rc, err := client.Get(context.Background(), "test-key")
if err != nil {
t.Fatalf("unexpected error getting key: %s", err)
@ -168,7 +167,7 @@ func TestS3GetReadNothingLoop(t *testing.T) {
}
s3Client := fullContentsS3API()
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
rc, err := client.Get(context.Background(), "test-key")
if err != nil {
t.Fatalf("unexpected error getting key: %s", err)
@ -213,7 +212,7 @@ func fullContentsS3API() *MockS3API {
func TestS3Upload(t *testing.T) {
s3Client := NewMockS3API()
uploaderClient := NewMockS3Uploader()
uploaderClient.UploadFunc.SetDefaultHook(func(ctx context.Context, input *s3manager.UploadInput) error {
uploaderClient.UploadFunc.SetDefaultHook(func(ctx context.Context, input *s3.PutObjectInput) error {
// Synchronously read the reader so that we trigger the
// counting reader inside the Upload method and test the
// count.
@ -257,13 +256,13 @@ func TestS3Combine(t *testing.T) {
s3Client.UploadPartCopyFunc.SetDefaultHook(func(ctx context.Context, input *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
return &s3.UploadPartCopyOutput{
CopyPartResult: &s3.CopyPartResult{
CopyPartResult: &s3types.CopyPartResult{
ETag: aws.String(fmt.Sprintf("etag-%s", *input.CopySource)),
},
}, nil
})
s3Client.HeadObjectFunc.SetDefaultReturn(&s3.HeadObjectOutput{ContentLength: aws.Int64(42)}, nil)
s3Client.HeadObjectFunc.SetDefaultReturn(&s3.HeadObjectOutput{ContentLength: int64(42)}, nil)
client := testS3Client(s3Client, nil)
@ -277,7 +276,7 @@ func TestS3Combine(t *testing.T) {
if calls := s3Client.UploadPartCopyFunc.History(); len(calls) != 3 {
t.Fatalf("unexpected number of UploadPartCopy calls. want=%d have=%d", 3, len(calls))
} else {
parts := map[int64]string{}
parts := map[int32]string{}
for _, call := range calls {
if value := *call.Arg1.Bucket; value != "test-bucket" {
t.Errorf("unexpected bucket argument. want=%s have=%s", "test-bucket", value)
@ -289,10 +288,10 @@ func TestS3Combine(t *testing.T) {
t.Errorf("unexpected key argument. want=%s have=%s", "uid", value)
}
parts[*call.Arg1.PartNumber] = *call.Arg1.CopySource
parts[call.Arg1.PartNumber] = *call.Arg1.CopySource
}
expectedParts := map[int64]string{
expectedParts := map[int32]string{
1: "test-bucket/test-src1",
2: "test-bucket/test-src2",
3: "test-bucket/test-src3",
@ -319,12 +318,12 @@ func TestS3Combine(t *testing.T) {
} else if value := *calls[0].Arg1.UploadId; value != "uid" {
t.Errorf("unexpected uploadId argument. want=%s have=%s", "uid", value)
} else {
parts := map[int64]string{}
parts := map[int32]string{}
for _, part := range calls[0].Arg1.MultipartUpload.Parts {
parts[*part.PartNumber] = *part.ETag
parts[part.PartNumber] = *part.ETag
}
expectedParts := map[int64]string{
expectedParts := map[int32]string{
1: "etag-test-bucket/test-src1",
2: "etag-test-bucket/test-src2",
3: "etag-test-bucket/test-src3",
@ -381,39 +380,28 @@ func TestS3Delete(t *testing.T) {
}
}
func TestS3Lifecycle(t *testing.T) {
s3Client := NewMockS3API()
client := rawS3Client(s3Client, nil)
if lifecycle := client.lifecycle(); lifecycle == nil || len(lifecycle.Rules) != 2 {
func TestS3BucketLifecycleConfiguration(t *testing.T) {
if lifecycle := s3BucketLifecycleConfiguration("s3", time.Hour*24*3); lifecycle == nil || len(lifecycle.Rules) != 2 {
t.Fatalf("unexpected lifecycle rules")
} else {
var objectExpiration *int64
var objectExpiration int32
for _, rule := range lifecycle.Rules {
if rule.Expiration != nil {
if value := rule.Expiration.Days; value != nil {
objectExpiration = value
}
objectExpiration = rule.Expiration.Days
}
}
if objectExpiration == nil {
t.Fatalf("expected object expiration to be configured")
} else if *objectExpiration != 3 {
t.Errorf("unexpected ttl for object expiration. want=%d have=%d", 3, *objectExpiration)
if objectExpiration != 3 {
t.Errorf("unexpected ttl for object expiration. want=%d have=%d", 3, objectExpiration)
}
var multipartExpiration *int64
var multipartExpiration int32
for _, rule := range lifecycle.Rules {
if rule.AbortIncompleteMultipartUpload != nil {
if value := rule.AbortIncompleteMultipartUpload.DaysAfterInitiation; value != nil {
multipartExpiration = value
}
multipartExpiration = rule.AbortIncompleteMultipartUpload.DaysAfterInitiation
}
}
if multipartExpiration == nil {
t.Fatalf("expected multipart upload expiration to be configured")
} else if *multipartExpiration != 3 {
t.Errorf("unexpected ttl for multipart upload expiration. want=%d have=%d", 3, *multipartExpiration)
if multipartExpiration != 3 {
t.Errorf("unexpected ttl for multipart upload expiration. want=%d have=%d", 3, multipartExpiration)
}
}
}
@ -423,5 +411,5 @@ func testS3Client(client s3API, uploader s3Uploader) Store {
}
func rawS3Client(client s3API, uploader s3Uploader) *s3Store {
return newS3WithClients(client, uploader, "test-bucket", time.Hour*24*3, true, newOperations(&observation.TestContext))
return newS3WithClients(client, uploader, "test-bucket", true, nil, newOperations(&observation.TestContext))
}

10
go.mod
View File

@ -14,8 +14,13 @@ require (
github.com/aphistic/sweet-junit v0.2.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/avelino/slugify v0.0.0-20180501145920-855f152bd774
github.com/aws/aws-sdk-go v1.29.15
github.com/aws/aws-sdk-go-v2 v0.20.0
github.com/aws/aws-sdk-go-v2 v1.2.1
github.com/aws/aws-sdk-go-v2/config v1.1.2
github.com/aws/aws-sdk-go-v2/credentials v1.1.2
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3
github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2
github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1
github.com/aws/smithy-go v1.2.0
github.com/beevik/etree v1.1.0
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff
github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
@ -96,7 +101,6 @@ require (
github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1
github.com/jackc/pgconn v1.8.0
github.com/jackc/pgx/v4 v4.10.0
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5
github.com/joho/godotenv v1.3.0
github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible

35
go.sum
View File

@ -138,11 +138,34 @@ github.com/avelino/slugify v0.0.0-20180501145920-855f152bd774/go.mod h1:5wi5YYOp
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs=
github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go-v2 v0.20.0 h1:/yefUjgMrda9PNFwWctBU63nL10CJMdBwkAmaQ4w4Hs=
github.com/aws/aws-sdk-go-v2 v0.20.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw=
github.com/aws/aws-sdk-go-v2 v1.2.1 h1:055XAi+MtmhyYX161p+jWRibkCb9YpI2ymXZiW1dwVY=
github.com/aws/aws-sdk-go-v2 v1.2.1/go.mod h1:hTQc/9pYq5bfFACIUY9tc/2SYWd9Vnmw+testmuQeRY=
github.com/aws/aws-sdk-go-v2/config v1.1.2 h1:H2r6cwMvvINFpEC55Y7jcNaR/oc7zYIChrG2497wmBI=
github.com/aws/aws-sdk-go-v2/config v1.1.2/go.mod h1:77yIk+qmCS/94JlxbwV1d+YEyu6Z8FBlCGcSz3TdM6A=
github.com/aws/aws-sdk-go-v2/credentials v1.1.2 h1:YoNqfhxAJGZI+lStIbqgx30UcCqQ86fr7FjTLUvrFOc=
github.com/aws/aws-sdk-go-v2/credentials v1.1.2/go.mod h1:hofjw//lM0XLplgvzPPMA7oD0doQU1QpaIK1nweEEWg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3 h1:d3bKAGy4XdJyK8hz3Nx3WJJ4TCmYp2498G4mFY5wly0=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3/go.mod h1:Zr1Mj+KUMGVQ+WJvTT68EZJxqhjiie2PWSPGEUPaNY0=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3 h1:vhRq0752KGBMmLnVessDOpt+5XEdzM87hhiuwGiEpqc=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3/go.mod h1:9tFvXNMet5TrBa2bMLhZBvenXs4qKMqiG1n0MNR4FFA=
github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2 h1:PIAYXYIBEBFaSvCNTiXsxakePOXC2eZ+wpxWhD2J4O4=
github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2/go.mod h1:9Y2XYeoa34KWUK9wae2Iuel8OLRxc00SGgIlGUCOoXE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2 h1:GO0pL4QvQmA0fXJe3MHVO+emtg31MYq5/8sebSWgE6A=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2/go.mod h1:bYl7lGFQQdHia3uMQH4p6ImnuOeDNeUoydoXM5x8Yzw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3 h1:dST4y8pZKZdTPs4uwXmGCJmpycz1SHKmCSIhf3GqHEo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3/go.mod h1:C50Z41fJaJ7WgaeeCulOGAU3q4+4se4B3uOPFdhBi2I=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1 h1:+WCVceRPiUsrui55mDByXOVremK1n3Hm8GnB4ZD3eco=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1/go.mod h1:B+fb+BFbja6obFOHYmYE4iUMdej9aM2VGSpgdU1pn0M=
github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1 h1:3qn6YVXpOCK9seQ8ZilDyMrhpEUaZNaJG8SXNiCvk+c=
github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1/go.mod h1:3xGOyhtPPD/WXJUljmb5+ZXhNyHa4h6wgL6mWOF6S0c=
github.com/aws/aws-sdk-go-v2/service/sso v1.1.2 h1:9BnjX/ALn5uLo2DbgkwMpUkPL1VLQVBXcjZxqJBhf44=
github.com/aws/aws-sdk-go-v2/service/sso v1.1.2/go.mod h1:5yU1oE3+CVYYLUsaHt2AVU3CJJZ6ER4pwsrRD1L2KSc=
github.com/aws/aws-sdk-go-v2/service/sts v1.1.2 h1:7Kxqov7uQeP8WUEO0iHz3j9Bh0E1rJrn6cf/OGfcDds=
github.com/aws/aws-sdk-go-v2/service/sts v1.1.2/go.mod h1:zu7rotIY9P4Aoc6ytqLP9jeYrECDHUODB5Gbp+BSHl8=
github.com/aws/smithy-go v1.2.0 h1:0PoGBWXkXDIyVdPaZW9gMhaGzj3UOAgTdiVoHuuZAFA=
github.com/aws/smithy-go v1.2.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
@ -831,8 +854,10 @@ github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:x
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 h1:lrdPtrORjGv1HbbEvKWDUAy97mPpFm4B8hp77tcCUJY=
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=

View File

@ -6,8 +6,8 @@ import (
"encoding/hex"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/awserr"
"github.com/aws/aws-sdk-go-v2/service/codecommit"
codecommittypes "github.com/aws/aws-sdk-go-v2/service/codecommit/types"
"github.com/aws/smithy-go"
"github.com/pkg/errors"
"github.com/sourcegraph/sourcegraph/internal/rcache"
@ -53,16 +53,15 @@ func IsNotFound(err error) bool {
if err == ErrNotFound || errors.Cause(err) == ErrNotFound {
return true
}
if e, ok := err.(awserr.Error); ok {
return e.Code() == codecommit.ErrCodeRepositoryDoesNotExistException
}
return false
var rdnee codecommittypes.RepositoryDoesNotExistException
return errors.As(err, &rdnee)
}
// IsUnauthorized reports whether err is a AWS CodeCommit API unauthorized error.
func IsUnauthorized(err error) bool {
if e, ok := err.(awserr.Error); ok {
return e.Code() == "SignatureDoesNotMatch"
var ae smithy.APIError
if errors.As(err, &ae) {
return ae.ErrorCode() == "SignatureDoesNotMatch"
}
return false
}

View File

@ -1,8 +1,6 @@
package awscodecommit
import (
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
)
@ -22,6 +20,6 @@ func ExternalRepoSpec(repo *Repository, serviceID string) api.ExternalRepoSpec {
//
// This value uniquely identifies the most specific namespace in which AWS CodeCommit repositories
// are defined.
func ServiceID(awsPartition endpoints.Partition, awsRegion endpoints.Region, awsAccountID string) string {
return "arn:" + awsPartition.ID() + ":codecommit:" + awsRegion.ID() + ":" + awsAccountID + ":"
func ServiceID(awsPartition, awsRegion, awsAccountID string) string {
return "arn:" + awsPartition + ":codecommit:" + awsRegion + ":" + awsAccountID + ":"
}

View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/service/codecommit"
codecommittypes "github.com/aws/aws-sdk-go-v2/service/codecommit/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
@ -132,10 +133,8 @@ func (c *Client) getRepositoryFromAPI(ctx context.Context, arn string) (*Reposit
repoName = arn[i+1:]
}
svc := codecommit.New(c.aws)
req := svc.GetRepositoryRequest(&codecommit.GetRepositoryInput{RepositoryName: &repoName})
req.SetContext(ctx)
result, err := req.Send(ctx)
svc := codecommit.NewFromConfig(c.aws)
result, err := svc.GetRepository(ctx, &codecommit.GetRepositoryInput{RepositoryName: &repoName})
if err != nil {
return nil, err
}
@ -154,19 +153,17 @@ func (c *Client) ListRepositories(ctx context.Context, nextToken string) (repos
}
}()
svc := codecommit.New(c.aws)
svc := codecommit.NewFromConfig(c.aws)
// List repositories.
listInput := codecommit.ListRepositoriesInput{
Order: codecommit.OrderEnumDescending,
SortBy: codecommit.SortByEnumLastModifiedDate,
Order: codecommittypes.OrderEnumDescending,
SortBy: codecommittypes.SortByEnumModifiedDate,
}
if nextToken != "" {
listInput.NextToken = &nextToken
}
listReq := svc.ListRepositoriesRequest(&listInput)
listReq.SetContext(ctx)
listResult, err := listReq.Send(ctx)
listResult, err := svc.ListRepositories(ctx, &listInput)
if err != nil {
return nil, "", err
}
@ -201,9 +198,7 @@ func (c *Client) ListRepositories(ctx context.Context, nextToken string) (repos
func (c *Client) getRepositories(ctx context.Context, svc *codecommit.Client, repositoryNames []string) ([]*Repository, error) {
getInput := codecommit.BatchGetRepositoriesInput{RepositoryNames: repositoryNames}
getReq := svc.BatchGetRepositoriesRequest(&getInput)
getReq.SetContext(ctx)
getResult, err := getReq.Send(ctx)
getResult, err := svc.BatchGetRepositories(ctx, &getInput)
if err != nil {
return nil, err
}
@ -243,7 +238,7 @@ func (w *wrappedError) Unauthorized() bool {
return IsUnauthorized(w.err)
}
func fromRepoMetadata(m *codecommit.RepositoryMetadata) *Repository {
func fromRepoMetadata(m *codecommittypes.RepositoryMetadata) *Repository {
repo := Repository{
ARN: *m.Arn,
AccountID: *m.AccountId,

View File

@ -7,9 +7,12 @@ import (
"net/url"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/defaults"
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
"github.com/aws/aws-sdk-go-v2/config"
awscredentials "github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/codecommit"
"github.com/inconshreveable/log15"
"github.com/pkg/errors"
"golang.org/x/net/http2"
"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
@ -27,9 +30,8 @@ type AWSCodeCommitSource struct {
svc *types.ExternalService
config *schema.AWSCodeCommitConnection
awsConfig aws.Config
awsPartition endpoints.Partition // "aws", "aws-cn", "aws-us-gov"
awsRegion endpoints.Region
awsPartition string // "aws", "aws-cn", "aws-us-gov"
awsRegion string
client *awscodecommit.Client
exclude excludeFunc
@ -45,22 +47,12 @@ func NewAWSCodeCommitSource(svc *types.ExternalService, cf *httpcli.Factory) (*A
}
func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitConnection, cf *httpcli.Factory) (*AWSCodeCommitSource, error) {
awsConfig := defaults.Config()
awsConfig.Region = c.Region
awsConfig.Credentials = aws.StaticCredentialsProvider{
Value: aws.Credentials{
AccessKeyID: c.AccessKeyID,
SecretAccessKey: c.SecretAccessKey,
Source: "sourcegraph-site-configuration",
},
}
if cf == nil {
cf = httpcli.NewExternalHTTPClientFactory()
}
cli, err := cf.Doer(func(c *http.Client) error {
tr := aws.NewBuildableHTTPClient().GetTransport()
tr := awshttp.NewBuildableClient().GetTransport()
if err := http2.ConfigureTransport(tr); err != nil {
return err
}
@ -72,7 +64,23 @@ func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitC
if err != nil {
return nil, err
}
awsConfig.HTTPClient = cli
awsConfig, err := config.LoadDefaultConfig(context.Background(),
config.WithRegion(c.Region),
config.WithCredentialsProvider(
awscredentials.StaticCredentialsProvider{
Value: aws.Credentials{
AccessKeyID: c.AccessKeyID,
SecretAccessKey: c.SecretAccessKey,
Source: "sourcegraph-site-configuration",
},
},
),
config.WithHTTPClient(cli),
)
if err != nil {
return nil, err
}
var eb excludeBuilder
for _, r := range c.Exclude {
@ -85,21 +93,18 @@ func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitC
}
s := &AWSCodeCommitSource{
svc: svc,
config: c,
awsConfig: awsConfig,
exclude: exclude,
client: awscodecommit.NewClient(awsConfig),
svc: svc,
config: c,
exclude: exclude,
client: awscodecommit.NewClient(awsConfig),
}
var ok bool
s.awsPartition, ok = endpoints.DefaultPartitions().ForRegion(c.Region)
if ok {
s.awsRegion, ok = s.awsPartition.Regions()[c.Region]
}
if !ok {
return nil, fmt.Errorf("unrecognized AWS region name: %q", c.Region)
endpoint, err := codecommit.NewDefaultEndpointResolver().ResolveEndpoint(c.Region, codecommit.EndpointResolverOptions{})
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("failed to resolve AWS region %q", c.Region))
}
s.awsPartition = endpoint.PartitionID
s.awsRegion = endpoint.SigningRegion
return s, nil
}