mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 17:11:49 +00:00
Upgrade aws-sdk-go-v2 (#19155)
This PR: - Migrates to the AWS v2 API in the codeintel s3 adapter - Upgrades the v2 package to the newest version
This commit is contained in:
parent
28c81089e8
commit
2a3bd2b515
@ -54,7 +54,7 @@ COPY --from=comby/comby:0.18.4@sha256:b47ce282778bfea7f80d45f5ef0cc546ba0d6347ba
|
||||
COPY --from=sourcegraph/syntect_server:331beda@sha256:6b8950b41993af3d10300b5a160fab1c06c337cb4614978f4fdb76e2588afbfe /syntect_server /usr/local/bin/
|
||||
|
||||
# install minio (keep this up to date with docker-images/minio/Dockerfile)
|
||||
ENV MINIO_VERSION=RELEASE.2020-10-18T21-54-12Z
|
||||
ENV MINIO_VERSION=RELEASE.2021-04-06T23-11-00Z
|
||||
RUN wget "https://dl.min.io/server/minio/release/linux-amd64/archive/minio.$MINIO_VERSION" && \
|
||||
chmod +x "minio.$MINIO_VERSION" && \
|
||||
mv "minio.$MINIO_VERSION" /usr/local/bin/minio
|
||||
|
||||
@ -14,9 +14,6 @@ go get -u all
|
||||
## names. This is causing our use of our k8s client to panic.
|
||||
go get github.com/golang/protobuf@v1.3.5
|
||||
|
||||
## Newer versions removed some types in the endpoint package we relied on. Unsure how to fix yet, so punting.
|
||||
go get github.com/aws/aws-sdk-go-v2@v0.20.0
|
||||
|
||||
# Cleanup and validate everything still works
|
||||
go mod tidy
|
||||
go test -short -failfast ./... >/dev/null
|
||||
|
||||
@ -346,3 +346,10 @@
|
||||
:why: Internal module
|
||||
:versions: []
|
||||
:when: 2021-03-09 19:42:12.214862934 Z
|
||||
- - :license
|
||||
- github.com/aws/smithy-go
|
||||
- Apache 2.0
|
||||
- :who:
|
||||
:why: Inference broken, LICENSE file lives at https://github.com/aws/smithy-go/blob/main/LICENSE
|
||||
:versions: []
|
||||
:when: 2021-04-14 12:51:19.503802000 Z
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
set -ex
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
|
||||
# Retag the 2020-10-18 MinIO release
|
||||
MINIO_RELEASE="RELEASE.2020-10-18T21-54-12Z"
|
||||
# Retag the 2021-04-06 MinIO release
|
||||
MINIO_RELEASE="RELEASE.2021-04-06T23-11-00Z"
|
||||
docker pull minio/minio:$MINIO_RELEASE
|
||||
docker tag minio/minio:$MINIO_RELEASE "$IMAGE"
|
||||
|
||||
@ -8,7 +8,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/inconshreveable/log15"
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@ -199,15 +199,6 @@ func initializeUploadStore(ctx context.Context, uploadStore uploadstore.Store) e
|
||||
}
|
||||
|
||||
func isRequestError(err error) bool {
|
||||
for err != nil {
|
||||
if e, ok := err.(awserr.Error); ok {
|
||||
if e.Code() == "RequestError" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
err = errors.Unwrap(err)
|
||||
}
|
||||
|
||||
return false
|
||||
var rse *smithyhttp.RequestSendError
|
||||
return errors.As(err, &rse)
|
||||
}
|
||||
|
||||
@ -1,8 +1,13 @@
|
||||
package uploadstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
)
|
||||
|
||||
func TestConfigDefaults(t *testing.T) {
|
||||
@ -70,25 +75,44 @@ func TestConfigS3(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigMinIOSessionOptions(t *testing.T) {
|
||||
func TestS3ClientOptions(t *testing.T) {
|
||||
config := Config{}
|
||||
config.SetMockGetter(mapGetter(nil))
|
||||
config.Load()
|
||||
|
||||
options := s3SessionOptions("minio", config.S3)
|
||||
// minIO
|
||||
{
|
||||
options := &s3.Options{}
|
||||
s3ClientOptions("minio", config.S3)(options)
|
||||
|
||||
if value := *options.Config.Region; value != "us-east-1" {
|
||||
t.Errorf("unexpected region. want=%s have=%s", "us-east-1", value)
|
||||
if options.EndpointResolver == nil {
|
||||
t.Fatalf("unexpected endpoint option")
|
||||
}
|
||||
endpoint, err := options.EndpointResolver.ResolveEndpoint("us-east-2", s3.EndpointResolverOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if endpoint.URL != "http://minio:9000" {
|
||||
t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", endpoint.URL)
|
||||
}
|
||||
|
||||
if !options.UsePathStyle {
|
||||
t.Errorf("invalid UsePathStyle setting for S3Options")
|
||||
}
|
||||
}
|
||||
if value := *options.Config.Endpoint; value != "http://minio:9000" {
|
||||
t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", value)
|
||||
}
|
||||
if options.Config.S3ForcePathStyle == nil || !*options.Config.S3ForcePathStyle {
|
||||
t.Errorf("expected path style option")
|
||||
|
||||
// S3
|
||||
{
|
||||
options := &s3.Options{}
|
||||
s3ClientOptions("s3", config.S3)(options)
|
||||
|
||||
if diff := cmp.Diff(&s3.Options{}, options); diff != "" {
|
||||
t.Fatalf("invalid s3 options returned for S3: %s", diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigS3SessionOptions(t *testing.T) {
|
||||
func TestS3ClientConfig(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"PRECISE_CODE_INTEL_UPLOAD_BACKEND": "S3",
|
||||
"PRECISE_CODE_INTEL_UPLOAD_BUCKET": "lsif-uploads",
|
||||
@ -104,16 +128,28 @@ func TestConfigS3SessionOptions(t *testing.T) {
|
||||
config.SetMockGetter(mapGetter(env))
|
||||
config.Load()
|
||||
|
||||
options := s3SessionOptions("s3", config.S3)
|
||||
cfg, err := s3ClientConfig(context.Background(), config.S3)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if value := *options.Config.Region; value != "us-east-2" {
|
||||
if value := cfg.Region; value != "us-east-2" {
|
||||
t.Errorf("unexpected region. want=%s have=%s", "us-east-2", value)
|
||||
}
|
||||
if options.Config.Endpoint != nil {
|
||||
t.Errorf("unexpected endpoint option")
|
||||
cred, err := cfg.Credentials.Retrieve(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if options.Config.S3ForcePathStyle != nil {
|
||||
t.Errorf("unexpected path style option")
|
||||
if diff := cmp.Diff(aws.Credentials{
|
||||
AccessKeyID: config.S3.AccessKeyID,
|
||||
SecretAccessKey: config.S3.SecretAccessKey,
|
||||
SessionToken: config.S3.SessionToken,
|
||||
Source: "StaticCredentials",
|
||||
}, cred); diff != "" {
|
||||
t.Errorf("invalid credential returned: %s", diff)
|
||||
}
|
||||
if cfg.EndpointResolver != nil {
|
||||
t.Errorf("unexpected endpoint option")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -6,8 +6,7 @@ import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
s3 "github.com/aws/aws-sdk-go/service/s3"
|
||||
s3manager "github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
s3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
// MockS3API is a mock implementation of the s3API interface (from the
|
||||
@ -1143,7 +1142,7 @@ type MockS3Uploader struct {
|
||||
func NewMockS3Uploader() *MockS3Uploader {
|
||||
return &MockS3Uploader{
|
||||
UploadFunc: &S3UploaderUploadFunc{
|
||||
defaultHook: func(context.Context, *s3manager.UploadInput) error {
|
||||
defaultHook: func(context.Context, *s3.PutObjectInput) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
@ -1155,7 +1154,7 @@ func NewMockS3Uploader() *MockS3Uploader {
|
||||
// github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/uploadstore).
|
||||
// It is redefined here as it is unexported in the source packge.
|
||||
type surrogateMockS3Uploader interface {
|
||||
Upload(context.Context, *s3manager.UploadInput) error
|
||||
Upload(context.Context, *s3.PutObjectInput) error
|
||||
}
|
||||
|
||||
// NewMockS3UploaderFrom creates a new mock of the MockS3Uploader interface.
|
||||
@ -1171,15 +1170,15 @@ func NewMockS3UploaderFrom(i surrogateMockS3Uploader) *MockS3Uploader {
|
||||
// S3UploaderUploadFunc describes the behavior when the Upload method of the
|
||||
// parent MockS3Uploader instance is invoked.
|
||||
type S3UploaderUploadFunc struct {
|
||||
defaultHook func(context.Context, *s3manager.UploadInput) error
|
||||
hooks []func(context.Context, *s3manager.UploadInput) error
|
||||
defaultHook func(context.Context, *s3.PutObjectInput) error
|
||||
hooks []func(context.Context, *s3.PutObjectInput) error
|
||||
history []S3UploaderUploadFuncCall
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// Upload delegates to the next hook function in the queue and stores the
|
||||
// parameter and result values of this invocation.
|
||||
func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3manager.UploadInput) error {
|
||||
func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3.PutObjectInput) error {
|
||||
r0 := m.UploadFunc.nextHook()(v0, v1)
|
||||
m.UploadFunc.appendCall(S3UploaderUploadFuncCall{v0, v1, r0})
|
||||
return r0
|
||||
@ -1187,7 +1186,7 @@ func (m *MockS3Uploader) Upload(v0 context.Context, v1 *s3manager.UploadInput) e
|
||||
|
||||
// SetDefaultHook sets function that is called when the Upload method of the
|
||||
// parent MockS3Uploader instance is invoked and the hook queue is empty.
|
||||
func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3manager.UploadInput) error) {
|
||||
func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3.PutObjectInput) error) {
|
||||
f.defaultHook = hook
|
||||
}
|
||||
|
||||
@ -1195,7 +1194,7 @@ func (f *S3UploaderUploadFunc) SetDefaultHook(hook func(context.Context, *s3mana
|
||||
// Upload method of the parent MockS3Uploader instance invokes the hook at
|
||||
// the front of the queue and discards it. After the queue is empty, the
|
||||
// default hook function is invoked for any future action.
|
||||
func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3manager.UploadInput) error) {
|
||||
func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3.PutObjectInput) error) {
|
||||
f.mutex.Lock()
|
||||
f.hooks = append(f.hooks, hook)
|
||||
f.mutex.Unlock()
|
||||
@ -1204,7 +1203,7 @@ func (f *S3UploaderUploadFunc) PushHook(hook func(context.Context, *s3manager.Up
|
||||
// SetDefaultReturn calls SetDefaultDefaultHook with a function that returns
|
||||
// the given values.
|
||||
func (f *S3UploaderUploadFunc) SetDefaultReturn(r0 error) {
|
||||
f.SetDefaultHook(func(context.Context, *s3manager.UploadInput) error {
|
||||
f.SetDefaultHook(func(context.Context, *s3.PutObjectInput) error {
|
||||
return r0
|
||||
})
|
||||
}
|
||||
@ -1212,12 +1211,12 @@ func (f *S3UploaderUploadFunc) SetDefaultReturn(r0 error) {
|
||||
// PushReturn calls PushDefaultHook with a function that returns the given
|
||||
// values.
|
||||
func (f *S3UploaderUploadFunc) PushReturn(r0 error) {
|
||||
f.PushHook(func(context.Context, *s3manager.UploadInput) error {
|
||||
f.PushHook(func(context.Context, *s3.PutObjectInput) error {
|
||||
return r0
|
||||
})
|
||||
}
|
||||
|
||||
func (f *S3UploaderUploadFunc) nextHook() func(context.Context, *s3manager.UploadInput) error {
|
||||
func (f *S3UploaderUploadFunc) nextHook() func(context.Context, *s3.PutObjectInput) error {
|
||||
f.mutex.Lock()
|
||||
defer f.mutex.Unlock()
|
||||
|
||||
@ -1255,7 +1254,7 @@ type S3UploaderUploadFuncCall struct {
|
||||
Arg0 context.Context
|
||||
// Arg1 is the value of the 2nd argument passed to this method
|
||||
// invocation.
|
||||
Arg1 *s3manager.UploadInput
|
||||
Arg1 *s3.PutObjectInput
|
||||
// Result0 is the value of the 1st result returned from this method
|
||||
// invocation.
|
||||
Result0 error
|
||||
|
||||
@ -3,8 +3,8 @@ package uploadstore
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type s3API interface {
|
||||
@ -20,52 +20,52 @@ type s3API interface {
|
||||
}
|
||||
|
||||
type s3Uploader interface {
|
||||
Upload(ctx context.Context, input *s3manager.UploadInput) error
|
||||
Upload(ctx context.Context, input *s3.PutObjectInput) error
|
||||
}
|
||||
|
||||
type s3APIShim struct{ *s3.S3 }
|
||||
type s3UploaderShim struct{ *s3manager.Uploader }
|
||||
type s3APIShim struct{ *s3.Client }
|
||||
type s3UploaderShim struct{ *manager.Uploader }
|
||||
|
||||
var _ s3API = &s3APIShim{}
|
||||
var _ s3Uploader = &s3UploaderShim{}
|
||||
|
||||
func (s *s3APIShim) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) (*s3.CreateBucketOutput, error) {
|
||||
return s.S3.CreateBucketWithContext(ctx, input)
|
||||
return s.Client.CreateBucket(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) PutBucketLifecycleConfiguration(ctx context.Context, input *s3.PutBucketLifecycleConfigurationInput) (*s3.PutBucketLifecycleConfigurationOutput, error) {
|
||||
return s.S3.PutBucketLifecycleConfigurationWithContext(ctx, input)
|
||||
return s.Client.PutBucketLifecycleConfiguration(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
return s.S3.HeadObjectWithContext(ctx, input)
|
||||
return s.Client.HeadObject(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
return s.S3.GetObjectWithContext(ctx, input)
|
||||
return s.Client.GetObject(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
|
||||
return s.S3.DeleteObjectWithContext(ctx, input)
|
||||
return s.Client.DeleteObject(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return s.S3.CreateMultipartUploadWithContext(ctx, input)
|
||||
return s.Client.CreateMultipartUpload(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) {
|
||||
return s.S3.AbortMultipartUploadWithContext(ctx, input)
|
||||
return s.Client.AbortMultipartUpload(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
|
||||
return s.S3.UploadPartCopyWithContext(ctx, input)
|
||||
return s.Client.UploadPartCopy(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3APIShim) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
return s.S3.CompleteMultipartUploadWithContext(ctx, input)
|
||||
return s.Client.CompleteMultipartUpload(ctx, input)
|
||||
}
|
||||
|
||||
func (s *s3UploaderShim) Upload(ctx context.Context, input *s3manager.UploadInput) error {
|
||||
_, err := s.Uploader.UploadWithContext(ctx, input)
|
||||
func (s *s3UploaderShim) Upload(ctx context.Context, input *s3.PutObjectInput) error {
|
||||
_, err := s.Uploader.Upload(ctx, input)
|
||||
return err
|
||||
}
|
||||
|
||||
@ -9,12 +9,12 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/inconshreveable/log15"
|
||||
"github.com/opentracing/opentracing-go/log"
|
||||
"github.com/pkg/errors"
|
||||
@ -25,12 +25,12 @@ import (
|
||||
)
|
||||
|
||||
type s3Store struct {
|
||||
bucket string
|
||||
ttl time.Duration
|
||||
manageBucket bool
|
||||
client s3API
|
||||
uploader s3Uploader
|
||||
operations *operations
|
||||
bucket string
|
||||
manageBucket bool
|
||||
client s3API
|
||||
uploader s3Uploader
|
||||
bucketLifecycleConfiguration *s3types.BucketLifecycleConfiguration
|
||||
operations *operations
|
||||
}
|
||||
|
||||
var _ Store = &s3Store{}
|
||||
@ -53,25 +53,25 @@ func (c *S3Config) load(parent *env.BaseConfig) {
|
||||
|
||||
// newS3FromConfig creates a new store backed by AWS Simple Storage Service.
|
||||
func newS3FromConfig(ctx context.Context, config *Config, operations *operations) (Store, error) {
|
||||
sess, err := session.NewSessionWithOptions(s3SessionOptions(config.Backend, config.S3))
|
||||
cfg, err := s3ClientConfig(ctx, config.S3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s3Client := s3.New(sess)
|
||||
s3Client := s3.NewFromConfig(cfg, s3ClientOptions(config.Backend, config.S3))
|
||||
api := &s3APIShim{s3Client}
|
||||
uploader := &s3UploaderShim{s3manager.NewUploaderWithClient(s3Client)}
|
||||
return newS3WithClients(api, uploader, config.Bucket, config.TTL, config.ManageBucket, operations), nil
|
||||
uploader := &s3UploaderShim{manager.NewUploader(s3Client)}
|
||||
return newS3WithClients(api, uploader, config.Bucket, config.ManageBucket, s3BucketLifecycleConfiguration(config.Backend, config.TTL), operations), nil
|
||||
}
|
||||
|
||||
func newS3WithClients(client s3API, uploader s3Uploader, bucket string, ttl time.Duration, manageBucket bool, operations *operations) *s3Store {
|
||||
func newS3WithClients(client s3API, uploader s3Uploader, bucket string, manageBucket bool, lifecycleConfiguration *s3types.BucketLifecycleConfiguration, operations *operations) *s3Store {
|
||||
return &s3Store{
|
||||
bucket: bucket,
|
||||
ttl: ttl,
|
||||
manageBucket: manageBucket,
|
||||
client: client,
|
||||
uploader: uploader,
|
||||
operations: operations,
|
||||
bucket: bucket,
|
||||
manageBucket: manageBucket,
|
||||
client: client,
|
||||
uploader: uploader,
|
||||
operations: operations,
|
||||
bucketLifecycleConfiguration: lifecycleConfiguration,
|
||||
}
|
||||
}
|
||||
|
||||
@ -166,7 +166,7 @@ func (s *s3Store) Upload(ctx context.Context, key string, r io.Reader) (_ int64,
|
||||
|
||||
cr := &countingReader{r: r}
|
||||
|
||||
if err := s.uploader.Upload(ctx, &s3manager.UploadInput{
|
||||
if err := s.uploader.Upload(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: cr,
|
||||
@ -220,7 +220,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
|
||||
Bucket: multipartUpload.Bucket,
|
||||
Key: multipartUpload.Key,
|
||||
UploadId: multipartUpload.UploadId,
|
||||
PartNumber: aws.Int64(int64(partNumber)),
|
||||
PartNumber: int32(partNumber),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", s.bucket, source)),
|
||||
})
|
||||
if err != nil {
|
||||
@ -236,13 +236,13 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var parts []*s3.CompletedPart
|
||||
var parts []s3types.CompletedPart
|
||||
for i := 0; i < len(sources); i++ {
|
||||
partNumber := i + 1
|
||||
|
||||
parts = append(parts, &s3.CompletedPart{
|
||||
parts = append(parts, s3types.CompletedPart{
|
||||
ETag: etags[partNumber],
|
||||
PartNumber: aws.Int64(int64(partNumber)),
|
||||
PartNumber: int32(partNumber),
|
||||
})
|
||||
}
|
||||
|
||||
@ -250,7 +250,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
|
||||
Bucket: multipartUpload.Bucket,
|
||||
Key: multipartUpload.Key,
|
||||
UploadId: multipartUpload.UploadId,
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{Parts: parts},
|
||||
MultipartUpload: &s3types.CompletedMultipartUpload{Parts: parts},
|
||||
}); err != nil {
|
||||
return 0, errors.Wrap(err, "failed to complete multipart upload")
|
||||
}
|
||||
@ -263,7 +263,7 @@ func (s *s3Store) Compose(ctx context.Context, destination string, sources ...st
|
||||
return 0, errors.Wrap(err, "failed to stat composed object")
|
||||
}
|
||||
|
||||
return *obj.ContentLength, nil
|
||||
return obj.ContentLength, nil
|
||||
}
|
||||
|
||||
func (s *s3Store) Delete(ctx context.Context, key string) (err error) {
|
||||
@ -285,15 +285,13 @@ func (s *s3Store) create(ctx context.Context) error {
|
||||
Bucket: aws.String(s.bucket),
|
||||
})
|
||||
|
||||
codes := []string{
|
||||
s3.ErrCodeBucketAlreadyExists,
|
||||
s3.ErrCodeBucketAlreadyOwnedByYou,
|
||||
var bae *s3types.BucketAlreadyExists
|
||||
if errors.As(err, &bae) {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, code := range codes {
|
||||
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == code {
|
||||
return nil
|
||||
}
|
||||
var baoby *s3types.BucketAlreadyOwnedByYou
|
||||
if errors.As(err, &baoby) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
@ -302,34 +300,13 @@ func (s *s3Store) create(ctx context.Context) error {
|
||||
func (s *s3Store) update(ctx context.Context) error {
|
||||
configureRequest := &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
LifecycleConfiguration: s.lifecycle(),
|
||||
LifecycleConfiguration: s.bucketLifecycleConfiguration,
|
||||
}
|
||||
|
||||
_, err := s.client.PutBucketLifecycleConfiguration(ctx, configureRequest)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *s3Store) lifecycle() *s3.BucketLifecycleConfiguration {
|
||||
days := aws.Int64(int64(s.ttl / (time.Hour * 24)))
|
||||
|
||||
return &s3.BucketLifecycleConfiguration{
|
||||
Rules: []*s3.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("Expiration Rule"),
|
||||
Status: aws.String("Enabled"),
|
||||
Filter: &s3.LifecycleRuleFilter{Prefix: aws.String("")},
|
||||
Expiration: &s3.LifecycleExpiration{Days: days},
|
||||
},
|
||||
{
|
||||
ID: aws.String("Abort Incomplete Multipart Upload Rule"),
|
||||
Status: aws.String("Enabled"),
|
||||
Filter: &s3.LifecycleRuleFilter{Prefix: aws.String("")},
|
||||
AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{DaysAfterInitiation: days},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *s3Store) deleteSources(ctx context.Context, bucket string, sources []string) error {
|
||||
return goroutine.RunWorkersOverStrings(sources, func(index int, source string) error {
|
||||
if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
@ -356,24 +333,26 @@ func (r *countingReader) Read(p []byte) (n int, err error) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
func s3SessionOptions(backend string, config S3Config) session.Options {
|
||||
creds := credentials.NewStaticCredentials(
|
||||
config.AccessKeyID,
|
||||
config.SecretAccessKey,
|
||||
config.SessionToken,
|
||||
)
|
||||
|
||||
awsConfig := aws.Config{
|
||||
Credentials: creds,
|
||||
Region: aws.String(config.Region),
|
||||
func s3ClientConfig(ctx context.Context, s3config S3Config) (aws.Config, error) {
|
||||
optFns := []func(*awsconfig.LoadOptions) error{
|
||||
awsconfig.WithRegion(s3config.Region),
|
||||
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
s3config.AccessKeyID,
|
||||
s3config.SecretAccessKey,
|
||||
s3config.SessionToken,
|
||||
)),
|
||||
}
|
||||
|
||||
if backend == "minio" {
|
||||
awsConfig.Endpoint = aws.String(config.Endpoint)
|
||||
awsConfig.S3ForcePathStyle = aws.Bool(true)
|
||||
}
|
||||
return awsconfig.LoadDefaultConfig(ctx, optFns...)
|
||||
}
|
||||
|
||||
return session.Options{Config: awsConfig}
|
||||
func s3ClientOptions(backend string, config S3Config) func(o *s3.Options) {
|
||||
return func(o *s3.Options) {
|
||||
if backend == "minio" {
|
||||
o.EndpointResolver = s3.EndpointResolverFromURL(config.Endpoint)
|
||||
o.UsePathStyle = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeToPipe invokes the given function with a pipe writer in a goroutine
|
||||
@ -391,3 +370,27 @@ func isConnectionResetError(err error) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func s3BucketLifecycleConfiguration(backend string, ttl time.Duration) *s3types.BucketLifecycleConfiguration {
|
||||
days := int32(ttl / (time.Hour * 24))
|
||||
|
||||
rules := []s3types.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("Expiration Rule"),
|
||||
Status: s3types.ExpirationStatusEnabled,
|
||||
Filter: &s3types.LifecycleRuleFilterMemberPrefix{Value: ""},
|
||||
Expiration: &s3types.LifecycleExpiration{Days: days},
|
||||
},
|
||||
}
|
||||
|
||||
if backend != "minio" {
|
||||
rules = append(rules, s3types.LifecycleRule{
|
||||
ID: aws.String("Abort Incomplete Multipart Upload Rule"),
|
||||
Status: s3types.ExpirationStatusEnabled,
|
||||
Filter: &s3types.LifecycleRuleFilterMemberPrefix{Value: ""},
|
||||
AbortIncompleteMultipartUpload: &s3types.AbortIncompleteMultipartUpload{DaysAfterInitiation: days},
|
||||
})
|
||||
}
|
||||
|
||||
return &s3types.BucketLifecycleConfiguration{Rules: rules}
|
||||
}
|
||||
|
||||
@ -12,10 +12,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
s3 "github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
s3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
@ -43,9 +42,9 @@ func TestS3Init(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestS3InitBucketExists(t *testing.T) {
|
||||
for _, code := range []string{s3.ErrCodeBucketAlreadyExists, s3.ErrCodeBucketAlreadyOwnedByYou} {
|
||||
for _, err := range []error{&s3types.BucketAlreadyExists{}, &s3types.BucketAlreadyOwnedByYou{}} {
|
||||
s3Client := NewMockS3API()
|
||||
s3Client.CreateBucketFunc.SetDefaultReturn(nil, awserr.New(code, "", nil))
|
||||
s3Client.CreateBucketFunc.SetDefaultReturn(nil, err)
|
||||
|
||||
client := testS3Client(s3Client, nil)
|
||||
if err := client.Init(context.Background()); err != nil {
|
||||
@ -68,7 +67,7 @@ func TestS3InitBucketExists(t *testing.T) {
|
||||
|
||||
func TestS3UnmanagedInit(t *testing.T) {
|
||||
s3Client := NewMockS3API()
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
|
||||
if err := client.Init(context.Background()); err != nil {
|
||||
t.Fatalf("unexpected error initializing client: %s", err)
|
||||
}
|
||||
@ -87,7 +86,7 @@ func TestS3Get(t *testing.T) {
|
||||
Body: ioutil.NopCloser(bytes.NewReader([]byte("TEST PAYLOAD"))),
|
||||
}, nil)
|
||||
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
|
||||
rc, err := client.Get(context.Background(), "test-key")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error getting key: %s", err)
|
||||
@ -139,7 +138,7 @@ func TestS3GetTransientErrors(t *testing.T) {
|
||||
}
|
||||
|
||||
s3Client := fullContentsS3API()
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
|
||||
rc, err := client.Get(context.Background(), "test-key")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error getting key: %s", err)
|
||||
@ -168,7 +167,7 @@ func TestS3GetReadNothingLoop(t *testing.T) {
|
||||
}
|
||||
|
||||
s3Client := fullContentsS3API()
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", time.Hour*24, false, newOperations(&observation.TestContext))
|
||||
client := newS3WithClients(s3Client, nil, "test-bucket", false, nil, newOperations(&observation.TestContext))
|
||||
rc, err := client.Get(context.Background(), "test-key")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error getting key: %s", err)
|
||||
@ -213,7 +212,7 @@ func fullContentsS3API() *MockS3API {
|
||||
func TestS3Upload(t *testing.T) {
|
||||
s3Client := NewMockS3API()
|
||||
uploaderClient := NewMockS3Uploader()
|
||||
uploaderClient.UploadFunc.SetDefaultHook(func(ctx context.Context, input *s3manager.UploadInput) error {
|
||||
uploaderClient.UploadFunc.SetDefaultHook(func(ctx context.Context, input *s3.PutObjectInput) error {
|
||||
// Synchronously read the reader so that we trigger the
|
||||
// counting reader inside the Upload method and test the
|
||||
// count.
|
||||
@ -257,13 +256,13 @@ func TestS3Combine(t *testing.T) {
|
||||
|
||||
s3Client.UploadPartCopyFunc.SetDefaultHook(func(ctx context.Context, input *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
|
||||
return &s3.UploadPartCopyOutput{
|
||||
CopyPartResult: &s3.CopyPartResult{
|
||||
CopyPartResult: &s3types.CopyPartResult{
|
||||
ETag: aws.String(fmt.Sprintf("etag-%s", *input.CopySource)),
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
|
||||
s3Client.HeadObjectFunc.SetDefaultReturn(&s3.HeadObjectOutput{ContentLength: aws.Int64(42)}, nil)
|
||||
s3Client.HeadObjectFunc.SetDefaultReturn(&s3.HeadObjectOutput{ContentLength: int64(42)}, nil)
|
||||
|
||||
client := testS3Client(s3Client, nil)
|
||||
|
||||
@ -277,7 +276,7 @@ func TestS3Combine(t *testing.T) {
|
||||
if calls := s3Client.UploadPartCopyFunc.History(); len(calls) != 3 {
|
||||
t.Fatalf("unexpected number of UploadPartCopy calls. want=%d have=%d", 3, len(calls))
|
||||
} else {
|
||||
parts := map[int64]string{}
|
||||
parts := map[int32]string{}
|
||||
for _, call := range calls {
|
||||
if value := *call.Arg1.Bucket; value != "test-bucket" {
|
||||
t.Errorf("unexpected bucket argument. want=%s have=%s", "test-bucket", value)
|
||||
@ -289,10 +288,10 @@ func TestS3Combine(t *testing.T) {
|
||||
t.Errorf("unexpected key argument. want=%s have=%s", "uid", value)
|
||||
}
|
||||
|
||||
parts[*call.Arg1.PartNumber] = *call.Arg1.CopySource
|
||||
parts[call.Arg1.PartNumber] = *call.Arg1.CopySource
|
||||
}
|
||||
|
||||
expectedParts := map[int64]string{
|
||||
expectedParts := map[int32]string{
|
||||
1: "test-bucket/test-src1",
|
||||
2: "test-bucket/test-src2",
|
||||
3: "test-bucket/test-src3",
|
||||
@ -319,12 +318,12 @@ func TestS3Combine(t *testing.T) {
|
||||
} else if value := *calls[0].Arg1.UploadId; value != "uid" {
|
||||
t.Errorf("unexpected uploadId argument. want=%s have=%s", "uid", value)
|
||||
} else {
|
||||
parts := map[int64]string{}
|
||||
parts := map[int32]string{}
|
||||
for _, part := range calls[0].Arg1.MultipartUpload.Parts {
|
||||
parts[*part.PartNumber] = *part.ETag
|
||||
parts[part.PartNumber] = *part.ETag
|
||||
}
|
||||
|
||||
expectedParts := map[int64]string{
|
||||
expectedParts := map[int32]string{
|
||||
1: "etag-test-bucket/test-src1",
|
||||
2: "etag-test-bucket/test-src2",
|
||||
3: "etag-test-bucket/test-src3",
|
||||
@ -381,39 +380,28 @@ func TestS3Delete(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3Lifecycle(t *testing.T) {
|
||||
s3Client := NewMockS3API()
|
||||
client := rawS3Client(s3Client, nil)
|
||||
|
||||
if lifecycle := client.lifecycle(); lifecycle == nil || len(lifecycle.Rules) != 2 {
|
||||
func TestS3BucketLifecycleConfiguration(t *testing.T) {
|
||||
if lifecycle := s3BucketLifecycleConfiguration("s3", time.Hour*24*3); lifecycle == nil || len(lifecycle.Rules) != 2 {
|
||||
t.Fatalf("unexpected lifecycle rules")
|
||||
} else {
|
||||
var objectExpiration *int64
|
||||
var objectExpiration int32
|
||||
for _, rule := range lifecycle.Rules {
|
||||
if rule.Expiration != nil {
|
||||
if value := rule.Expiration.Days; value != nil {
|
||||
objectExpiration = value
|
||||
}
|
||||
objectExpiration = rule.Expiration.Days
|
||||
}
|
||||
}
|
||||
if objectExpiration == nil {
|
||||
t.Fatalf("expected object expiration to be configured")
|
||||
} else if *objectExpiration != 3 {
|
||||
t.Errorf("unexpected ttl for object expiration. want=%d have=%d", 3, *objectExpiration)
|
||||
if objectExpiration != 3 {
|
||||
t.Errorf("unexpected ttl for object expiration. want=%d have=%d", 3, objectExpiration)
|
||||
}
|
||||
|
||||
var multipartExpiration *int64
|
||||
var multipartExpiration int32
|
||||
for _, rule := range lifecycle.Rules {
|
||||
if rule.AbortIncompleteMultipartUpload != nil {
|
||||
if value := rule.AbortIncompleteMultipartUpload.DaysAfterInitiation; value != nil {
|
||||
multipartExpiration = value
|
||||
}
|
||||
multipartExpiration = rule.AbortIncompleteMultipartUpload.DaysAfterInitiation
|
||||
}
|
||||
}
|
||||
if multipartExpiration == nil {
|
||||
t.Fatalf("expected multipart upload expiration to be configured")
|
||||
} else if *multipartExpiration != 3 {
|
||||
t.Errorf("unexpected ttl for multipart upload expiration. want=%d have=%d", 3, *multipartExpiration)
|
||||
if multipartExpiration != 3 {
|
||||
t.Errorf("unexpected ttl for multipart upload expiration. want=%d have=%d", 3, multipartExpiration)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -423,5 +411,5 @@ func testS3Client(client s3API, uploader s3Uploader) Store {
|
||||
}
|
||||
|
||||
func rawS3Client(client s3API, uploader s3Uploader) *s3Store {
|
||||
return newS3WithClients(client, uploader, "test-bucket", time.Hour*24*3, true, newOperations(&observation.TestContext))
|
||||
return newS3WithClients(client, uploader, "test-bucket", true, nil, newOperations(&observation.TestContext))
|
||||
}
|
||||
|
||||
10
go.mod
10
go.mod
@ -14,8 +14,13 @@ require (
|
||||
github.com/aphistic/sweet-junit v0.2.0 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
|
||||
github.com/avelino/slugify v0.0.0-20180501145920-855f152bd774
|
||||
github.com/aws/aws-sdk-go v1.29.15
|
||||
github.com/aws/aws-sdk-go-v2 v0.20.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.2.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.1.2
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.1.2
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3
|
||||
github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1
|
||||
github.com/aws/smithy-go v1.2.0
|
||||
github.com/beevik/etree v1.1.0
|
||||
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff
|
||||
github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
|
||||
@ -96,7 +101,6 @@ require (
|
||||
github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1
|
||||
github.com/jackc/pgconn v1.8.0
|
||||
github.com/jackc/pgx/v4 v4.10.0
|
||||
github.com/jmespath/go-jmespath v0.3.0 // indirect
|
||||
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5
|
||||
github.com/joho/godotenv v1.3.0
|
||||
github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible
|
||||
|
||||
35
go.sum
35
go.sum
@ -138,11 +138,34 @@ github.com/avelino/slugify v0.0.0-20180501145920-855f152bd774/go.mod h1:5wi5YYOp
|
||||
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
|
||||
github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs=
|
||||
github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
|
||||
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
|
||||
github.com/aws/aws-sdk-go-v2 v0.20.0 h1:/yefUjgMrda9PNFwWctBU63nL10CJMdBwkAmaQ4w4Hs=
|
||||
github.com/aws/aws-sdk-go-v2 v0.20.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.2.1 h1:055XAi+MtmhyYX161p+jWRibkCb9YpI2ymXZiW1dwVY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.2.1/go.mod h1:hTQc/9pYq5bfFACIUY9tc/2SYWd9Vnmw+testmuQeRY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.1.2 h1:H2r6cwMvvINFpEC55Y7jcNaR/oc7zYIChrG2497wmBI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.1.2/go.mod h1:77yIk+qmCS/94JlxbwV1d+YEyu6Z8FBlCGcSz3TdM6A=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.1.2 h1:YoNqfhxAJGZI+lStIbqgx30UcCqQ86fr7FjTLUvrFOc=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.1.2/go.mod h1:hofjw//lM0XLplgvzPPMA7oD0doQU1QpaIK1nweEEWg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3 h1:d3bKAGy4XdJyK8hz3Nx3WJJ4TCmYp2498G4mFY5wly0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3/go.mod h1:Zr1Mj+KUMGVQ+WJvTT68EZJxqhjiie2PWSPGEUPaNY0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3 h1:vhRq0752KGBMmLnVessDOpt+5XEdzM87hhiuwGiEpqc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.0.3/go.mod h1:9tFvXNMet5TrBa2bMLhZBvenXs4qKMqiG1n0MNR4FFA=
|
||||
github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2 h1:PIAYXYIBEBFaSvCNTiXsxakePOXC2eZ+wpxWhD2J4O4=
|
||||
github.com/aws/aws-sdk-go-v2/service/codecommit v1.1.2/go.mod h1:9Y2XYeoa34KWUK9wae2Iuel8OLRxc00SGgIlGUCOoXE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2 h1:GO0pL4QvQmA0fXJe3MHVO+emtg31MYq5/8sebSWgE6A=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.2/go.mod h1:bYl7lGFQQdHia3uMQH4p6ImnuOeDNeUoydoXM5x8Yzw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3 h1:dST4y8pZKZdTPs4uwXmGCJmpycz1SHKmCSIhf3GqHEo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3/go.mod h1:C50Z41fJaJ7WgaeeCulOGAU3q4+4se4B3uOPFdhBi2I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1 h1:+WCVceRPiUsrui55mDByXOVremK1n3Hm8GnB4ZD3eco=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.1.1/go.mod h1:B+fb+BFbja6obFOHYmYE4iUMdej9aM2VGSpgdU1pn0M=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1 h1:3qn6YVXpOCK9seQ8ZilDyMrhpEUaZNaJG8SXNiCvk+c=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.2.1/go.mod h1:3xGOyhtPPD/WXJUljmb5+ZXhNyHa4h6wgL6mWOF6S0c=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.1.2 h1:9BnjX/ALn5uLo2DbgkwMpUkPL1VLQVBXcjZxqJBhf44=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.1.2/go.mod h1:5yU1oE3+CVYYLUsaHt2AVU3CJJZ6ER4pwsrRD1L2KSc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.1.2 h1:7Kxqov7uQeP8WUEO0iHz3j9Bh0E1rJrn6cf/OGfcDds=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.1.2/go.mod h1:zu7rotIY9P4Aoc6ytqLP9jeYrECDHUODB5Gbp+BSHl8=
|
||||
github.com/aws/smithy-go v1.2.0 h1:0PoGBWXkXDIyVdPaZW9gMhaGzj3UOAgTdiVoHuuZAFA=
|
||||
github.com/aws/smithy-go v1.2.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
||||
@ -831,8 +854,10 @@ github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:x
|
||||
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
|
||||
github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
|
||||
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 h1:lrdPtrORjGv1HbbEvKWDUAy97mPpFm4B8hp77tcCUJY=
|
||||
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||
|
||||
@ -6,8 +6,8 @@ import (
|
||||
"encoding/hex"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go-v2/service/codecommit"
|
||||
codecommittypes "github.com/aws/aws-sdk-go-v2/service/codecommit/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/internal/rcache"
|
||||
@ -53,16 +53,15 @@ func IsNotFound(err error) bool {
|
||||
if err == ErrNotFound || errors.Cause(err) == ErrNotFound {
|
||||
return true
|
||||
}
|
||||
if e, ok := err.(awserr.Error); ok {
|
||||
return e.Code() == codecommit.ErrCodeRepositoryDoesNotExistException
|
||||
}
|
||||
return false
|
||||
var rdnee codecommittypes.RepositoryDoesNotExistException
|
||||
return errors.As(err, &rdnee)
|
||||
}
|
||||
|
||||
// IsUnauthorized reports whether err is a AWS CodeCommit API unauthorized error.
|
||||
func IsUnauthorized(err error) bool {
|
||||
if e, ok := err.(awserr.Error); ok {
|
||||
return e.Code() == "SignatureDoesNotMatch"
|
||||
var ae smithy.APIError
|
||||
if errors.As(err, &ae) {
|
||||
return ae.ErrorCode() == "SignatureDoesNotMatch"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
package awscodecommit
|
||||
|
||||
import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/internal/api"
|
||||
"github.com/sourcegraph/sourcegraph/internal/extsvc"
|
||||
)
|
||||
@ -22,6 +20,6 @@ func ExternalRepoSpec(repo *Repository, serviceID string) api.ExternalRepoSpec {
|
||||
//
|
||||
// This value uniquely identifies the most specific namespace in which AWS CodeCommit repositories
|
||||
// are defined.
|
||||
func ServiceID(awsPartition endpoints.Partition, awsRegion endpoints.Region, awsAccountID string) string {
|
||||
return "arn:" + awsPartition.ID() + ":codecommit:" + awsRegion.ID() + ":" + awsAccountID + ":"
|
||||
func ServiceID(awsPartition, awsRegion, awsAccountID string) string {
|
||||
return "arn:" + awsPartition + ":codecommit:" + awsRegion + ":" + awsAccountID + ":"
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/codecommit"
|
||||
codecommittypes "github.com/aws/aws-sdk-go-v2/service/codecommit/types"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
@ -132,10 +133,8 @@ func (c *Client) getRepositoryFromAPI(ctx context.Context, arn string) (*Reposit
|
||||
repoName = arn[i+1:]
|
||||
}
|
||||
|
||||
svc := codecommit.New(c.aws)
|
||||
req := svc.GetRepositoryRequest(&codecommit.GetRepositoryInput{RepositoryName: &repoName})
|
||||
req.SetContext(ctx)
|
||||
result, err := req.Send(ctx)
|
||||
svc := codecommit.NewFromConfig(c.aws)
|
||||
result, err := svc.GetRepository(ctx, &codecommit.GetRepositoryInput{RepositoryName: &repoName})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -154,19 +153,17 @@ func (c *Client) ListRepositories(ctx context.Context, nextToken string) (repos
|
||||
}
|
||||
}()
|
||||
|
||||
svc := codecommit.New(c.aws)
|
||||
svc := codecommit.NewFromConfig(c.aws)
|
||||
|
||||
// List repositories.
|
||||
listInput := codecommit.ListRepositoriesInput{
|
||||
Order: codecommit.OrderEnumDescending,
|
||||
SortBy: codecommit.SortByEnumLastModifiedDate,
|
||||
Order: codecommittypes.OrderEnumDescending,
|
||||
SortBy: codecommittypes.SortByEnumModifiedDate,
|
||||
}
|
||||
if nextToken != "" {
|
||||
listInput.NextToken = &nextToken
|
||||
}
|
||||
listReq := svc.ListRepositoriesRequest(&listInput)
|
||||
listReq.SetContext(ctx)
|
||||
listResult, err := listReq.Send(ctx)
|
||||
listResult, err := svc.ListRepositories(ctx, &listInput)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
@ -201,9 +198,7 @@ func (c *Client) ListRepositories(ctx context.Context, nextToken string) (repos
|
||||
|
||||
func (c *Client) getRepositories(ctx context.Context, svc *codecommit.Client, repositoryNames []string) ([]*Repository, error) {
|
||||
getInput := codecommit.BatchGetRepositoriesInput{RepositoryNames: repositoryNames}
|
||||
getReq := svc.BatchGetRepositoriesRequest(&getInput)
|
||||
getReq.SetContext(ctx)
|
||||
getResult, err := getReq.Send(ctx)
|
||||
getResult, err := svc.BatchGetRepositories(ctx, &getInput)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -243,7 +238,7 @@ func (w *wrappedError) Unauthorized() bool {
|
||||
return IsUnauthorized(w.err)
|
||||
}
|
||||
|
||||
func fromRepoMetadata(m *codecommit.RepositoryMetadata) *Repository {
|
||||
func fromRepoMetadata(m *codecommittypes.RepositoryMetadata) *Repository {
|
||||
repo := Repository{
|
||||
ARN: *m.Arn,
|
||||
AccountID: *m.AccountId,
|
||||
|
||||
@ -7,9 +7,12 @@ import (
|
||||
"net/url"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/defaults"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/endpoints"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
awscredentials "github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/codecommit"
|
||||
"github.com/inconshreveable/log15"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
|
||||
@ -27,9 +30,8 @@ type AWSCodeCommitSource struct {
|
||||
svc *types.ExternalService
|
||||
config *schema.AWSCodeCommitConnection
|
||||
|
||||
awsConfig aws.Config
|
||||
awsPartition endpoints.Partition // "aws", "aws-cn", "aws-us-gov"
|
||||
awsRegion endpoints.Region
|
||||
awsPartition string // "aws", "aws-cn", "aws-us-gov"
|
||||
awsRegion string
|
||||
client *awscodecommit.Client
|
||||
|
||||
exclude excludeFunc
|
||||
@ -45,22 +47,12 @@ func NewAWSCodeCommitSource(svc *types.ExternalService, cf *httpcli.Factory) (*A
|
||||
}
|
||||
|
||||
func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitConnection, cf *httpcli.Factory) (*AWSCodeCommitSource, error) {
|
||||
awsConfig := defaults.Config()
|
||||
awsConfig.Region = c.Region
|
||||
awsConfig.Credentials = aws.StaticCredentialsProvider{
|
||||
Value: aws.Credentials{
|
||||
AccessKeyID: c.AccessKeyID,
|
||||
SecretAccessKey: c.SecretAccessKey,
|
||||
Source: "sourcegraph-site-configuration",
|
||||
},
|
||||
}
|
||||
|
||||
if cf == nil {
|
||||
cf = httpcli.NewExternalHTTPClientFactory()
|
||||
}
|
||||
|
||||
cli, err := cf.Doer(func(c *http.Client) error {
|
||||
tr := aws.NewBuildableHTTPClient().GetTransport()
|
||||
tr := awshttp.NewBuildableClient().GetTransport()
|
||||
if err := http2.ConfigureTransport(tr); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -72,7 +64,23 @@ func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitC
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
awsConfig.HTTPClient = cli
|
||||
|
||||
awsConfig, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithRegion(c.Region),
|
||||
config.WithCredentialsProvider(
|
||||
awscredentials.StaticCredentialsProvider{
|
||||
Value: aws.Credentials{
|
||||
AccessKeyID: c.AccessKeyID,
|
||||
SecretAccessKey: c.SecretAccessKey,
|
||||
Source: "sourcegraph-site-configuration",
|
||||
},
|
||||
},
|
||||
),
|
||||
config.WithHTTPClient(cli),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var eb excludeBuilder
|
||||
for _, r := range c.Exclude {
|
||||
@ -85,21 +93,18 @@ func newAWSCodeCommitSource(svc *types.ExternalService, c *schema.AWSCodeCommitC
|
||||
}
|
||||
|
||||
s := &AWSCodeCommitSource{
|
||||
svc: svc,
|
||||
config: c,
|
||||
awsConfig: awsConfig,
|
||||
exclude: exclude,
|
||||
client: awscodecommit.NewClient(awsConfig),
|
||||
svc: svc,
|
||||
config: c,
|
||||
exclude: exclude,
|
||||
client: awscodecommit.NewClient(awsConfig),
|
||||
}
|
||||
|
||||
var ok bool
|
||||
s.awsPartition, ok = endpoints.DefaultPartitions().ForRegion(c.Region)
|
||||
if ok {
|
||||
s.awsRegion, ok = s.awsPartition.Regions()[c.Region]
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unrecognized AWS region name: %q", c.Region)
|
||||
endpoint, err := codecommit.NewDefaultEndpointResolver().ResolveEndpoint(c.Region, codecommit.EndpointResolverOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, fmt.Sprintf("failed to resolve AWS region %q", c.Region))
|
||||
}
|
||||
s.awsPartition = endpoint.PartitionID
|
||||
s.awsRegion = endpoint.SigningRegion
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user