diff --git a/go.mod b/go.mod index ddc4df46590..9992d1d79e0 100644 --- a/go.mod +++ b/go.mod @@ -540,7 +540,7 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/scim2/filter-parser/v2 v2.2.0 github.com/snabb/diagio v1.0.0 // indirect - github.com/sourcegraph/conc v0.3.0 + github.com/sourcegraph/conc v0.2.0 github.com/sourcegraph/mountinfo v0.0.0-20230106004439-7026e28cef67 github.com/sourcegraph/sourcegraph/monitoring v0.0.0-20230124144931-b2d81b1accb6 github.com/sourcegraph/zoekt v0.0.0-20230825171831-40a9a23bb04b diff --git a/go.sum b/go.sum index 376db2e2926..f6aa6e1e401 100644 --- a/go.sum +++ b/go.sum @@ -1995,8 +1995,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sourcegraph/alertmanager v0.21.1-0.20211110092431-863f5b1ee51b h1:Mlytsllyx6d1eaKXt8urXX0YjP5Tq4UGV+BfL6Yc1aQ= github.com/sourcegraph/alertmanager v0.21.1-0.20211110092431-863f5b1ee51b/go.mod h1:0MLTrjQI8EuVmvykEhcfr/7X0xmaDAZrqMgxIq3OXHk= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/sourcegraph/conc v0.2.0 h1:96VpOCAtXDCQ8Oycz0ftHqdPyMi8w12ltN4L2noYg7s= +github.com/sourcegraph/conc v0.2.0/go.mod h1:8lmPpTLA0hsWqw4lw7wS1e694U2tMjRrc1Asvupb4QM= github.com/sourcegraph/embedded-postgres v1.19.1-0.20230624001757-345a8df15ded h1:QcxHhicvH6TFpSmC3vZKWbwLSHmwy72+CESqjjaIsZA= github.com/sourcegraph/embedded-postgres v1.19.1-0.20230624001757-345a8df15ded/go.mod h1:0B+3bPsMvcNgR9nN+bdM2x9YaNYDnf3ksUqYp1OAub0= github.com/sourcegraph/go-ctags v0.0.0-20230111110657-c27675da7f71 h1:tsWE3F3StWvnwLnC4JWb0zX0UHY9GULQtu/aoQvLJvI= diff --git a/lib/codeintel/upload/BUILD.bazel b/lib/codeintel/upload/BUILD.bazel index 1d537090261..a4fe4af0f45 100644 --- a/lib/codeintel/upload/BUILD.bazel +++ b/lib/codeintel/upload/BUILD.bazel @@ -19,7 +19,6 @@ go_library( "//lib/errors", "//lib/output", "@com_github_klauspost_pgzip//:pgzip", - "@com_github_sourcegraph_conc//pool", "@com_github_sourcegraph_scip//bindings/go/scip", "@org_golang_google_protobuf//proto", ], diff --git a/lib/codeintel/upload/upload.go b/lib/codeintel/upload/upload.go index 097990d1b81..a677f03ebaa 100644 --- a/lib/codeintel/upload/upload.go +++ b/lib/codeintel/upload/upload.go @@ -7,8 +7,6 @@ import ( "os" "sync" - "github.com/sourcegraph/conc/pool" - "github.com/sourcegraph/sourcegraph/lib/errors" "github.com/sourcegraph/sourcegraph/lib/output" ) @@ -212,15 +210,18 @@ func uploadMultipartIndexParts(ctx context.Context, httpClient Client, opts Uplo ) defer func() { complete(err) }() - pool := new(pool.ErrorPool).WithFirstError().WithContext(ctx) - if opts.MaxConcurrency > 0 { - pool.WithMaxGoroutines(opts.MaxConcurrency) - } + var wg sync.WaitGroup + errs := make(chan error, len(readers)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() for i, reader := range readers { - i, reader := i, reader + wg.Add(1) + + go func(i int, reader io.ReadSeeker) { + defer wg.Done() - pool.Go(func(ctx context.Context) error { // Determine size of this reader. If we're not the last reader in the slice, // then we're the maximum payload size. Otherwise, we're whatever is left. partReaderLen := opts.MaxPayloadSizeBytes @@ -235,16 +236,25 @@ func uploadMultipartIndexParts(ctx context.Context, httpClient Client, opts Uplo } if err := uploadIndexFile(ctx, httpClient, opts, reader, partReaderLen, requestOptions, progress, retry, i, len(readers)); err != nil { - return err + errs <- err + cancel() } else if progress != nil { // Mark complete in case we debounced our last updates progress.SetValue(i, 1) } - return nil - }) + }(i, reader) } - return pool.Wait() + wg.Wait() + close(errs) + + for err := range errs { + if err != nil { + return err + } + } + + return nil } // uploadMultipartIndexFinalize performs the request to stitch the uploaded parts together and diff --git a/lib/codeintel/upload/upload_options.go b/lib/codeintel/upload/upload_options.go index e8f498399e8..b755fc19ec0 100644 --- a/lib/codeintel/upload/upload_options.go +++ b/lib/codeintel/upload/upload_options.go @@ -26,7 +26,6 @@ type SourcegraphInstanceOptions struct { MaxRetries int // The maximum number of retries per request RetryInterval time.Duration // Sleep duration between retries MaxPayloadSizeBytes int64 // The maximum number of bytes sent in a single request - MaxConcurrency int // The maximum number of concurrent uploads. Only relevant for multipart uploads GitHubToken string // GitHub token used for auth when lsif.enforceAuth is true (optional) GitLabToken string // GitLab token used for auth when lsif.enforceAuth is true (optional) HTTPClient Client diff --git a/lib/go.mod b/lib/go.mod index 5ca672dafa6..9e6cae15019 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -23,7 +23,6 @@ require ( github.com/mitchellh/copystructure v1.2.0 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 github.com/muesli/termenv v0.12.0 - github.com/sourcegraph/conc v0.3.0 github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3 github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf github.com/sourcegraph/log v0.0.0-20221206163500-7d93c6ad7037 @@ -114,7 +113,7 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/goleak v1.2.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.7.0 // indirect golang.org/x/mod v0.8.0 // indirect diff --git a/lib/go.sum b/lib/go.sum index 46d12a2e7aa..8534eb5d709 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -489,8 +489,6 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3 h1:11miag7hlORpW7ici5mL7T9PyiEsmVmf+8PFOvJ/ZrA= github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf h1:oAdWFqhStsWiiMP/vkkHiMXqFXzl1XfUNOdxKJbd6bI= @@ -578,13 +576,14 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=