mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 17:51:57 +00:00
Revert "codeintel: allow configuring max concurrency in uploads" (#56275)
Revert "codeintel: allow configuring max concurrency in uploads (#56117)"
This reverts commit de4d5471e3.
This commit is contained in:
Jean-Hadrien Chabran
GitHub
parent
54005e461f
commit
29f74f622b
2
go.mod
2
go.mod
@ -540,7 +540,7 @@ require (
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
|
||||
github.com/scim2/filter-parser/v2 v2.2.0
|
||||
github.com/snabb/diagio v1.0.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0
|
||||
github.com/sourcegraph/conc v0.2.0
|
||||
github.com/sourcegraph/mountinfo v0.0.0-20230106004439-7026e28cef67
|
||||
github.com/sourcegraph/sourcegraph/monitoring v0.0.0-20230124144931-b2d81b1accb6
|
||||
github.com/sourcegraph/zoekt v0.0.0-20230825171831-40a9a23bb04b
|
||||
|
||||
4
go.sum
4
go.sum
@ -1995,8 +1995,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
|
||||
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
|
||||
github.com/sourcegraph/alertmanager v0.21.1-0.20211110092431-863f5b1ee51b h1:Mlytsllyx6d1eaKXt8urXX0YjP5Tq4UGV+BfL6Yc1aQ=
|
||||
github.com/sourcegraph/alertmanager v0.21.1-0.20211110092431-863f5b1ee51b/go.mod h1:0MLTrjQI8EuVmvykEhcfr/7X0xmaDAZrqMgxIq3OXHk=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/sourcegraph/conc v0.2.0 h1:96VpOCAtXDCQ8Oycz0ftHqdPyMi8w12ltN4L2noYg7s=
|
||||
github.com/sourcegraph/conc v0.2.0/go.mod h1:8lmPpTLA0hsWqw4lw7wS1e694U2tMjRrc1Asvupb4QM=
|
||||
github.com/sourcegraph/embedded-postgres v1.19.1-0.20230624001757-345a8df15ded h1:QcxHhicvH6TFpSmC3vZKWbwLSHmwy72+CESqjjaIsZA=
|
||||
github.com/sourcegraph/embedded-postgres v1.19.1-0.20230624001757-345a8df15ded/go.mod h1:0B+3bPsMvcNgR9nN+bdM2x9YaNYDnf3ksUqYp1OAub0=
|
||||
github.com/sourcegraph/go-ctags v0.0.0-20230111110657-c27675da7f71 h1:tsWE3F3StWvnwLnC4JWb0zX0UHY9GULQtu/aoQvLJvI=
|
||||
|
||||
@ -19,7 +19,6 @@ go_library(
|
||||
"//lib/errors",
|
||||
"//lib/output",
|
||||
"@com_github_klauspost_pgzip//:pgzip",
|
||||
"@com_github_sourcegraph_conc//pool",
|
||||
"@com_github_sourcegraph_scip//bindings/go/scip",
|
||||
"@org_golang_google_protobuf//proto",
|
||||
],
|
||||
|
||||
@ -7,8 +7,6 @@ import (
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/sourcegraph/conc/pool"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/lib/errors"
|
||||
"github.com/sourcegraph/sourcegraph/lib/output"
|
||||
)
|
||||
@ -212,15 +210,18 @@ func uploadMultipartIndexParts(ctx context.Context, httpClient Client, opts Uplo
|
||||
)
|
||||
defer func() { complete(err) }()
|
||||
|
||||
pool := new(pool.ErrorPool).WithFirstError().WithContext(ctx)
|
||||
if opts.MaxConcurrency > 0 {
|
||||
pool.WithMaxGoroutines(opts.MaxConcurrency)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, len(readers))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
for i, reader := range readers {
|
||||
i, reader := i, reader
|
||||
wg.Add(1)
|
||||
|
||||
go func(i int, reader io.ReadSeeker) {
|
||||
defer wg.Done()
|
||||
|
||||
pool.Go(func(ctx context.Context) error {
|
||||
// Determine size of this reader. If we're not the last reader in the slice,
|
||||
// then we're the maximum payload size. Otherwise, we're whatever is left.
|
||||
partReaderLen := opts.MaxPayloadSizeBytes
|
||||
@ -235,16 +236,25 @@ func uploadMultipartIndexParts(ctx context.Context, httpClient Client, opts Uplo
|
||||
}
|
||||
|
||||
if err := uploadIndexFile(ctx, httpClient, opts, reader, partReaderLen, requestOptions, progress, retry, i, len(readers)); err != nil {
|
||||
return err
|
||||
errs <- err
|
||||
cancel()
|
||||
} else if progress != nil {
|
||||
// Mark complete in case we debounced our last updates
|
||||
progress.SetValue(i, 1)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}(i, reader)
|
||||
}
|
||||
|
||||
return pool.Wait()
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadMultipartIndexFinalize performs the request to stitch the uploaded parts together and
|
||||
|
||||
@ -26,7 +26,6 @@ type SourcegraphInstanceOptions struct {
|
||||
MaxRetries int // The maximum number of retries per request
|
||||
RetryInterval time.Duration // Sleep duration between retries
|
||||
MaxPayloadSizeBytes int64 // The maximum number of bytes sent in a single request
|
||||
MaxConcurrency int // The maximum number of concurrent uploads. Only relevant for multipart uploads
|
||||
GitHubToken string // GitHub token used for auth when lsif.enforceAuth is true (optional)
|
||||
GitLabToken string // GitLab token used for auth when lsif.enforceAuth is true (optional)
|
||||
HTTPClient Client
|
||||
|
||||
@ -23,7 +23,6 @@ require (
|
||||
github.com/mitchellh/copystructure v1.2.0
|
||||
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
|
||||
github.com/muesli/termenv v0.12.0
|
||||
github.com/sourcegraph/conc v0.3.0
|
||||
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3
|
||||
github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
|
||||
github.com/sourcegraph/log v0.0.0-20221206163500-7d93c6ad7037
|
||||
@ -114,7 +113,7 @@ require (
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/goleak v1.2.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/crypto v0.7.0 // indirect
|
||||
golang.org/x/mod v0.8.0 // indirect
|
||||
|
||||
@ -489,8 +489,6 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3 h1:11miag7hlORpW7ici5mL7T9PyiEsmVmf+8PFOvJ/ZrA=
|
||||
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
|
||||
github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf h1:oAdWFqhStsWiiMP/vkkHiMXqFXzl1XfUNOdxKJbd6bI=
|
||||
@ -578,13 +576,14 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
|
||||
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
|
||||
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
|
||||
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
|
||||
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user