add sourcegraph/codeintelutils into sourcegraph/sourcegraph (#18665)

* fold in codeintelutils

* remove remaining refs to codeintelutils

enterprise/cmd/frontend/internal/codeintel/httpapi/upload_handler.go

@@ -11,11 +11,11 @@ import (
 	"strconv"
 
 	"github.com/inconshreveable/log15"
-	"github.com/sourcegraph/codeintelutils"
 
 	"github.com/sourcegraph/sourcegraph/cmd/frontend/backend"
 	store "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/dbstore"
 	"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/uploadstore"
+	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/utils"
 	"github.com/sourcegraph/sourcegraph/internal/api"
 	"github.com/sourcegraph/sourcegraph/internal/conf"
 	"github.com/sourcegraph/sourcegraph/internal/errcode"

enterprise/lib/codeintel/utils/README.md

@@ -0,0 +1,3 @@
+# Codeintel Utilities
+
+This repo contains common functionality shared between a Sourcegraph instance and src-cli.

enterprise/lib/codeintel/utils/gzip.go

@@ -0,0 +1,31 @@
+package codeintelutils
+
+import (
+	"compress/gzip"
+	"io"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+// Gzip decorates a source reader by gzip compressing its contents.
+func Gzip(source io.Reader) io.Reader {
+	r, w := io.Pipe()
+	go func() {
+		// propagate gzip write errors into new reader
+		w.CloseWithError(gzipPipe(source, w))
+	}()
+	return r
+}
+
+// gzipPipe reads uncompressed data from r and writes compressed data to w.
+func gzipPipe(r io.Reader, w io.Writer) (err error) {
+	gzipWriter := gzip.NewWriter(w)
+	defer func() {
+		if closeErr := gzipWriter.Close(); closeErr != nil {
+			err = multierror.Append(err, err)
+		}
+	}()
+
+	_, err = io.Copy(gzipWriter, r)
+	return err
+}

enterprise/lib/codeintel/utils/gzip_test.go

@@ -0,0 +1,34 @@
+package codeintelutils
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestGzip(t *testing.T) {
+	var uncompressed []byte
+	for i := 0; i < 20000; i++ {
+		uncompressed = append(uncompressed, byte(i))
+	}
+
+	contents, err := ioutil.ReadAll(Gzip(bytes.NewReader(uncompressed)))
+	if err != nil {
+		t.Fatalf("unexpected error reading from gzip reader: %s", err)
+	}
+
+	gzipReader, err := gzip.NewReader(bytes.NewReader(contents))
+	if err != nil {
+		t.Fatalf("unexpected error creating gzip.Reader: %s", err)
+	}
+	decompressed, err := ioutil.ReadAll(gzipReader)
+	if err != nil {
+		t.Fatalf("unexpected error reading from gzip.Reader: %s", err)
+	}
+	if diff := cmp.Diff(decompressed, uncompressed); diff != "" {
+		t.Errorf("unexpected gzipped contents (-want +got):\n%s", diff)
+	}
+}

enterprise/lib/codeintel/utils/indexer_name.go

@@ -0,0 +1,56 @@
+package codeintelutils
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+
+	"github.com/pkg/errors"
+)
+
+// MaxBufferSize is the maximum size of the metaData line in the dump. This should be large enough
+// to be able to read the output of lsif-tsc for most cases, which will contain all glob-expanded
+// file names in the indexing of JavaScript projects.
+//
+// Data point: lodash's metaData vertex constructed by the args `*.js test/*.js --AllowJs --checkJs`
+// is 10639 characters long.
+const MaxBufferSize = 128 * 1024
+
+// ErrMetadataExceedsBuffer occurs when the first line of an LSIF index is too long to read.
+var ErrMetadataExceedsBuffer = errors.New("metaData vertex exceeds buffer")
+
+// ErrInvalidMetaDataVertex occurs when the first line of an LSIF index is not a valid metadata vertex.
+var ErrInvalidMetaDataVertex = errors.New("invalid metaData vertex")
+
+type metaDataVertex struct {
+	Label    string   `json:"label"`
+	ToolInfo toolInfo `json:"toolInfo"`
+}
+
+type toolInfo struct {
+	Name string `json:"name"`
+}
+
+// ReadIndexerName returns the name of the tool that generated the given index contents.
+// This function reads only the first line of the file, where the metadata vertex is
+// assumed to be in all valid dumps.
+func ReadIndexerName(r io.Reader) (string, error) {
+	line, isPrefix, err := bufio.NewReaderSize(r, MaxBufferSize).ReadLine()
+	if err != nil {
+		return "", err
+	}
+	if isPrefix {
+		return "", ErrMetadataExceedsBuffer
+	}
+
+	meta := metaDataVertex{}
+	if err := json.Unmarshal(line, &meta); err != nil {
+		return "", ErrInvalidMetaDataVertex
+	}
+
+	if meta.Label != "metaData" || meta.ToolInfo.Name == "" {
+		return "", ErrInvalidMetaDataVertex
+	}
+
+	return meta.ToolInfo.Name, nil
+}

enterprise/lib/codeintel/utils/indexer_name_test.go

@@ -0,0 +1,38 @@
+package codeintelutils
+
+import (
+	"bytes"
+	"io"
+	"strings"
+	"testing"
+)
+
+const testMetaDataVertex = `{"label": "metaData", "toolInfo": {"name": "test"}}`
+const testVertex = `{"id": "a", "type": "edge", "label": "textDocument/references", "outV": "b", "inV": "c"}`
+
+func TestReadIndexerName(t *testing.T) {
+	name, err := ReadIndexerName(generateTestIndex(testMetaDataVertex))
+	if err != nil {
+		t.Fatalf("unexpected error reading indexer name: %s", err)
+	}
+	if name != "test" {
+		t.Errorf("unexpected indexer name. want=%s have=%s", "test", name)
+	}
+}
+
+func TestReadIndexerNameMalformed(t *testing.T) {
+	for _, metaDataVertex := range []string{`invalid json`, `{"label": "textDocument/references"}`} {
+		if _, err := ReadIndexerName(generateTestIndex(metaDataVertex)); err != ErrInvalidMetaDataVertex {
+			t.Fatalf("unexpected error reading indexer name. want=%q have=%q", ErrInvalidMetaDataVertex, err)
+		}
+	}
+}
+
+func generateTestIndex(metaDataVertex string) io.Reader {
+	lines := []string{metaDataVertex}
+	for i := 0; i < 20000; i++ {
+		lines = append(lines, testVertex)
+	}
+
+	return bytes.NewReader([]byte(strings.Join(lines, "\n") + "\n"))
+}

enterprise/lib/codeintel/utils/rate.go

@@ -0,0 +1,32 @@
+package codeintelutils
+
+import (
+	"io"
+	"sync/atomic"
+)
+
+type rateReader struct {
+	reader io.Reader
+	read   int64
+	size   int64
+}
+
+func newRateReader(r io.Reader, size int64) *rateReader {
+	if r == nil {
+		return nil
+	}
+	return &rateReader{
+		reader: r,
+		size:   size,
+	}
+}
+
+func (r *rateReader) Read(p []byte) (int, error) {
+	n, err := r.reader.Read(p)
+	atomic.AddInt64(&r.read, int64(n))
+	return n, err
+}
+
+func (r *rateReader) Progress() float64 {
+	return float64(atomic.LoadInt64(&r.read)) / float64(r.size)
+}

enterprise/lib/codeintel/utils/split_file.go

@@ -0,0 +1,133 @@
+package codeintelutils
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+// SplitFile writes the contents of the given file into a series of temporary files, each of which
+// are gzipped and are no larger than the given max payload size. The file names are returned in the
+// order in which they were written. The cleanup function removes all temporary files, and wraps the
+// error argument with any additional errors that happen during cleanup.
+func SplitFile(filename string, maxPayloadSize int) (files []string, _ func(error) error, err error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer f.Close()
+
+	return SplitReaderIntoFiles(f, maxPayloadSize)
+}
+
+// SplitReaderIntoFiles writes the contents of the given reader into a series of temporary files, each of which
+// are gzipped and are no larger than the given max payload size. The file names are returned in the
+// order in which they were written. The cleanup function removes all temporary files, and wraps the
+// error argument with any additional errors that happen during cleanup.
+func SplitReaderIntoFiles(r io.ReaderAt, maxPayloadSize int) (files []string, _ func(error) error, err error) {
+	cleanup := func(err error) error {
+		for _, file := range files {
+			if removeErr := os.Remove(file); removeErr != nil {
+				err = multierror.Append(err, removeErr)
+			}
+		}
+
+		return err
+	}
+	defer func() {
+		if err != nil {
+			err = cleanup(err)
+		}
+	}()
+
+	// Create a function that returns a reader of the "next" chunk on each invocation.
+	makeNextReader := SplitReader(r, maxPayloadSize)
+
+	for {
+		partFile, err := ioutil.TempFile("", "")
+		if err != nil {
+			return nil, nil, err
+		}
+		defer partFile.Close()
+
+		n, err := io.Copy(partFile, makeNextReader())
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if n == 0 {
+			// File must be closed before it's removed (on Windows), so
+			// don't wait for the defer above. Double closing a file is
+			// fine, but the second close will return an ErrClosed value,
+			// which we aren't checking anyway.
+			_ = partFile.Close()
+
+			// Edge case: previous io.CopyN call returned err=nil and
+			// n=maxPayloadSize. Nothing written to this file so we
+			// can just undo its creation and fall-through to the break.
+			if err := os.Remove(partFile.Name()); err != nil {
+				return nil, nil, err
+			}
+
+			// Reader was empty, nothing left to do in this loop
+			break
+		}
+
+		files = append(files, partFile.Name())
+	}
+
+	return files, cleanup, nil
+}
+
+// SplitReader returns a function that returns readers that return a slice of the reader
+// at maxPayloadSize bytes long. Each reader returned will operate on the next sequential
+// slice from the source reader.
+func SplitReader(r io.ReaderAt, maxPayloadSize int) func() io.Reader {
+	offset := 0
+
+	return func() io.Reader {
+		pr, pw := io.Pipe()
+
+		go func() {
+			n, err := readAtN(pw, r, offset, maxPayloadSize)
+			offset += n
+			pw.CloseWithError(err)
+		}()
+
+		return pr
+	}
+}
+
+// readAtN reads n bytes (or until EOF) from the given ReaderAt starting at the given offset.
+// Each chunk read from the reader is written to the writer. Errors are forwarded from both
+// the reader and writer.
+func readAtN(w io.Writer, r io.ReaderAt, offset, n int) (int, error) {
+	buf := make([]byte, 32*1024) // same as used by io.Copy/copyBuffer
+
+	totalRead := 0
+	for n > 0 {
+		if n < len(buf) {
+			buf = buf[:n]
+		}
+
+		read, readErr := r.ReadAt(buf, int64(offset+totalRead))
+		if readErr != nil && readErr != io.EOF {
+			return totalRead, readErr
+		}
+
+		n -= read
+		totalRead += read
+
+		if _, writeErr := io.Copy(w, bytes.NewReader(buf[:read])); writeErr != nil {
+			return totalRead, writeErr
+		}
+		if readErr != nil {
+			return totalRead, readErr
+		}
+	}
+
+	return totalRead, nil
+}

enterprise/lib/codeintel/utils/split_file_test.go

@@ -0,0 +1,60 @@
+package codeintelutils
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestSplitFile(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("unexpected error creating temp file: %s", err)
+	}
+	defer func() { os.Remove(f.Name()) }()
+
+	var expectedContents []byte
+	for i := 0; i < 50; i++ {
+		var partContents []byte
+		for j := 0; j < 20000; j++ {
+			partContents = append(partContents, byte(j))
+		}
+
+		expectedContents = append(expectedContents, partContents...)
+		_, _ = f.Write(partContents)
+	}
+	f.Close()
+
+	files, cleanup, err := SplitFile(f.Name(), 18000)
+	if err != nil {
+		t.Fatalf("unexpected error splitting file: %s", err)
+	}
+
+	var contents []byte
+	for _, file := range files {
+		temp, err := ioutil.ReadFile(file)
+		if err != nil {
+			t.Fatalf("unexpected error reading file: %s", err)
+		}
+
+		if len(temp) > 18000 {
+			t.Errorf("chunk too large: want<%d have=%d", 18000, len(temp))
+		}
+
+		contents = append(contents, temp...)
+	}
+
+	if diff := cmp.Diff(expectedContents, contents); diff != "" {
+		t.Errorf("unexpected split contents (-want +got):\n%s", diff)
+	}
+
+	cleanup(nil)
+
+	for _, file := range files {
+		if _, err := os.Stat(file); !os.IsNotExist(err) {
+			t.Errorf("unexpected error. want=%q have=%q", os.ErrNotExist, err)
+		}
+	}
+}

enterprise/lib/codeintel/utils/stitch_files.go

@@ -0,0 +1,130 @@
+package codeintelutils
+
+import (
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+// PartFilenameFunc constructs the name of a part file from its part index.
+type PartFilenameFunc func(index int) string
+
+// StitchFiles combines multiple compressed file parts into a single file. Each part on disk be concatenated
+// into a single file. The content of each part is written to the new file sequentially. If decompress is
+// true, then each file part is gunzipped while read. If compress is true, the new file will be gzipped. On
+// success, the part files are removed.
+func StitchFiles(filename string, makePartFilename PartFilenameFunc, decompress, compress bool) error {
+	if err := os.MkdirAll(filepath.Dir(filename), os.ModePerm); err != nil {
+		return err
+	}
+
+	targetFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if closeErr := targetFile.Close(); closeErr != nil {
+			err = multierror.Append(err, closeErr)
+		}
+	}()
+
+	r, err := StitchFilesReader(makePartFilename, decompress)
+	if err != nil {
+		return err
+	}
+
+	if compress {
+		r = Gzip(r)
+	}
+
+	_, err = io.Copy(targetFile, r)
+	return err
+}
+
+// StitchFilesReader combines multiple compressed file parts into a single reader. Each part on disk is
+// concatenated into a single file. The content of each part is decompressed and written to returned
+// reader sequentially. On success, the part files are removed.
+func StitchFilesReader(makePartFilename PartFilenameFunc, decompress bool) (io.Reader, error) {
+	pr, pw := io.Pipe()
+
+	go func() {
+		defer pw.Close()
+
+		index := 0
+		for {
+			ok, err := writePart(pw, makePartFilename(index), decompress)
+			if err != nil {
+				_ = pw.CloseWithError(err)
+				return
+			}
+			if !ok {
+				break
+			}
+
+			index++
+		}
+
+		for i := index - 1; i >= 0; i-- {
+			_ = os.Remove(makePartFilename(i))
+		}
+	}()
+
+	return pr, nil
+}
+
+// WritePart opens the given filename and writes its content to the given writer.
+// Returns a boolean flag indicating whether or not a file was opened for reading.
+func writePart(w io.Writer, filename string, decompress bool) (bool, error) {
+	exists, reader, err := openPart(filename, decompress)
+	if err != nil || !exists {
+		return false, err
+	}
+	defer reader.Close()
+
+	_, err = io.Copy(w, reader)
+	return true, err
+}
+
+// openPart opens a gzip reader for a upload part file as well as a boolean flag
+// indicating if the file exists.
+func openPart(filename string, decompress bool) (bool, io.ReadCloser, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil, nil
+		}
+
+		return false, nil, err
+	}
+
+	if !decompress {
+		return true, f, nil
+	}
+
+	reader, err := gzip.NewReader(f)
+	if err != nil {
+		return false, nil, err
+	}
+
+	return true, &partReader{reader, f}, nil
+}
+
+// partReader bundles a gzip reader with its underlying reader. This overrides the
+// Close method on the gzip reader so that it also closes the underlying reader.
+type partReader struct {
+	*gzip.Reader
+	rc io.ReadCloser
+}
+
+func (r *partReader) Close() error {
+	for _, err := range []error{r.Reader.Close(), r.rc.Close()} {
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

enterprise/lib/codeintel/utils/stitch_files_test.go

@@ -0,0 +1,82 @@
+package codeintelutils
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestStitchMultipart(t *testing.T) {
+	for _, compress := range []bool{true, false} {
+		tempDir, err := ioutil.TempDir("", "codeintel-")
+		if err != nil {
+			t.Fatalf("unexpected error creating temp directory: %s", err)
+		}
+		defer os.RemoveAll(tempDir)
+
+		filename := filepath.Join(tempDir, "target")
+		partFilename := func(i int) string {
+			return filepath.Join(tempDir, fmt.Sprintf("%d.part", i))
+		}
+
+		var expectedContents []byte
+		for i := 0; i < 50; i++ {
+			partContents := make([]byte, 20000)
+			for j := 0; j < 20000; j++ {
+				partContents[j] = byte(i)
+			}
+			expectedContents = append(expectedContents, partContents...)
+
+			var buf bytes.Buffer
+			gzipWriter := gzip.NewWriter(&buf)
+
+			if _, err := io.Copy(gzipWriter, bytes.NewReader(partContents)); err != nil {
+				t.Fatalf("unexpected error writing to buffer: %s", err)
+			}
+			if err := gzipWriter.Close(); err != nil {
+				t.Fatalf("unexpected error closing gzip writer: %s", err)
+			}
+
+			if err := ioutil.WriteFile(partFilename(i), buf.Bytes(), os.ModePerm); err != nil {
+				t.Fatalf("unexpected error writing file: %s", err)
+			}
+		}
+
+		if err := StitchFiles(filename, partFilename, compress, compress); err != nil {
+			t.Fatalf("unexpected error stitching files: %s", err)
+		}
+
+		f, err := os.Open(filename)
+		if err != nil {
+			t.Fatalf("unexpected error opening file: %s", err)
+		}
+		defer f.Close()
+
+		gzipReader, err := gzip.NewReader(f)
+		if err != nil {
+			t.Fatalf("unexpected error opening gzip reader: %s", err)
+		}
+
+		contents, err := ioutil.ReadAll(gzipReader)
+		if err != nil {
+			t.Fatalf("unexpected error reading file: %s", err)
+		}
+
+		if diff := cmp.Diff(expectedContents, contents); diff != "" {
+			t.Errorf("unexpected file contents (-want +got):\n%s", diff)
+		}
+
+		for i := 0; i < 50; i++ {
+			if _, err := os.Stat(partFilename(i)); !os.IsNotExist(err) {
+				t.Errorf("unexpected error. want=%q have=%q", os.ErrNotExist, err)
+			}
+		}
+	}
+}

enterprise/lib/codeintel/utils/upload.go

@@ -0,0 +1,470 @@
+package codeintelutils
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+)
+
+type UploadIndexOpts struct {
+	Endpoint             string
+	Path                 string
+	AccessToken          string
+	AdditionalHeaders    map[string]string
+	Repo                 string
+	Commit               string
+	Root                 string
+	Indexer              string
+	GitHubToken          string
+	File                 string
+	AssociatedIndexID    *int
+	MaxPayloadSizeBytes  int
+	MaxRetries           int
+	RetryInterval        time.Duration
+	UploadProgressEvents chan UploadProgressEvent
+	Logger               RequestLogger
+}
+
+type UploadProgressEvent struct {
+	NumParts      int
+	Part          int
+	Progress      float64
+	TotalProgress float64
+}
+
+type RequestLogger interface {
+	LogRequest(req *http.Request)
+	LogResponse(req *http.Request, resp *http.Response, body []byte, elapsed time.Duration)
+}
+
+// ErrUnauthorized occurs when the upload endpoint returns a 401 response.
+var ErrUnauthorized = errors.New("unauthorized upload")
+
+// UploadIndex uploads the given index file to the upload endpoint. If the upload
+// file is large, it may be split into multiple chunks locally and uploaded over
+// multiple requests.
+func UploadIndex(opts UploadIndexOpts) (int, error) {
+	fileInfo, err := os.Stat(opts.File)
+	if err != nil {
+		return 0, err
+	}
+
+	if fileInfo.Size() <= int64(opts.MaxPayloadSizeBytes) {
+		id, err := uploadIndex(opts)
+		if err != nil {
+			return 0, err
+		}
+		return id, nil
+	}
+
+	id, err := uploadMultipartIndex(opts)
+	if err != nil {
+		return 0, err
+	}
+	return id, nil
+}
+
+// uploadIndex performs a single request to the upload endpoint. The new upload id is returned.
+func uploadIndex(opts UploadIndexOpts) (id int, err error) {
+	baseURL, err := makeBaseURL(opts)
+	if err != nil {
+		return 0, err
+	}
+
+	retry := makeRetry(opts.MaxRetries, opts.RetryInterval)
+
+	args := requestArgs{
+		baseURL:           baseURL,
+		accessToken:       opts.AccessToken,
+		additionalHeaders: opts.AdditionalHeaders,
+		repo:              opts.Repo,
+		commit:            opts.Commit,
+		root:              opts.Root,
+		indexer:           opts.Indexer,
+		gitHubToken:       opts.GitHubToken,
+		associatedIndexID: opts.AssociatedIndexID,
+	}
+
+	if err := retry(func() (_ bool, err error) {
+		return uploadFile(args, opts.File, false, &id, 0, 1, opts.UploadProgressEvents, opts.Logger)
+	}); err != nil {
+		return 0, err
+	}
+
+	return id, nil
+}
+
+// uploadMultipartIndex splits the index file into chunks small enough to upload, then
+// performs a series of request to the upload endpoint. The new upload id is returned.
+func uploadMultipartIndex(opts UploadIndexOpts) (id int, err error) {
+	baseURL, err := makeBaseURL(opts)
+	if err != nil {
+		return 0, err
+	}
+
+	retry := makeRetry(opts.MaxRetries, opts.RetryInterval)
+
+	compressedFile, err := compressFile(opts.File)
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		_ = os.Remove(compressedFile)
+	}()
+
+	files, cleanup, err := SplitFile(compressedFile, opts.MaxPayloadSizeBytes)
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		err = cleanup(err)
+	}()
+
+	setupArgs := requestArgs{
+		baseURL:           baseURL,
+		accessToken:       opts.AccessToken,
+		additionalHeaders: opts.AdditionalHeaders,
+		repo:              opts.Repo,
+		commit:            opts.Commit,
+		root:              opts.Root,
+		indexer:           opts.Indexer,
+		gitHubToken:       opts.GitHubToken,
+		multiPart:         true,
+		numParts:          len(files),
+		associatedIndexID: opts.AssociatedIndexID,
+	}
+
+	if err := retry(func() (bool, error) { return makeUploadRequest(setupArgs, nil, &id, opts.Logger) }); err != nil {
+		return 0, err
+	}
+
+	for i, file := range files {
+		uploadArgs := requestArgs{
+			baseURL:           baseURL,
+			accessToken:       opts.AccessToken,
+			additionalHeaders: opts.AdditionalHeaders,
+			uploadID:          id,
+			index:             i,
+		}
+
+		if err := retry(func() (_ bool, err error) {
+			return uploadFile(uploadArgs, file, true, nil, i, len(files), opts.UploadProgressEvents, opts.Logger)
+		}); err != nil {
+			return 0, err
+		}
+	}
+
+	finalizeArgs := requestArgs{
+		baseURL:           baseURL,
+		accessToken:       opts.AccessToken,
+		additionalHeaders: opts.AdditionalHeaders,
+		uploadID:          id,
+		done:              true,
+	}
+
+	if err := retry(func() (bool, error) { return makeUploadRequest(finalizeArgs, nil, nil, opts.Logger) }); err != nil {
+		return 0, err
+	}
+
+	return id, nil
+}
+
+func makeBaseURL(opts UploadIndexOpts) (*url.URL, error) {
+	endpointAndPath := opts.Endpoint
+	if opts.Path != "" {
+		endpointAndPath += opts.Path
+	} else {
+		endpointAndPath += "/.api/lsif/upload"
+	}
+
+	return url.Parse(endpointAndPath)
+}
+
+func compressFile(filename string) (_ string, err error) {
+	rawFile, err := os.Open(filename)
+	if err != nil {
+		return "", nil
+	}
+	defer rawFile.Close()
+
+	compressedFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		if closeErr := compressedFile.Close(); err != nil {
+			err = multierror.Append(err, closeErr)
+		}
+	}()
+
+	gzipWriter := gzip.NewWriter(compressedFile)
+	defer func() {
+		if closeErr := gzipWriter.Close(); err != nil {
+			err = multierror.Append(err, closeErr)
+		}
+	}()
+
+	if _, err := io.Copy(gzipWriter, rawFile); err != nil {
+		return "", nil
+	}
+
+	return compressedFile.Name(), nil
+}
+
+// requestArgs are a superset of the values that can be supplied in the query string of the
+// upload endpoint. The endpoint and access token fields must be set on every request, but the
+// remaining fields must be set when appropriate by the caller of makeUploadRequest.
+type requestArgs struct {
+	baseURL           *url.URL
+	accessToken       string
+	additionalHeaders map[string]string
+	repo              string
+	commit            string
+	root              string
+	indexer           string
+	gitHubToken       string
+	multiPart         bool
+	numParts          int
+	uploadID          int
+	index             int
+	done              bool
+	associatedIndexID *int
+}
+
+// EncodeQuery constructs a query string from the args.
+func (args requestArgs) EncodeQuery() string {
+	qs := newQueryValues()
+	qs.SetOptionalString("repository", args.repo)
+	qs.SetOptionalString("commit", args.commit)
+	qs.SetOptionalString("root", args.root)
+	qs.SetOptionalString("indexerName", args.indexer)
+	qs.SetOptionalString("github_token", args.gitHubToken)
+	qs.SetOptionalBool("multiPart", args.multiPart)
+	qs.SetOptionalInt("numParts", args.numParts)
+	qs.SetOptionalInt("uploadId", args.uploadID)
+	qs.SetOptionalBool("done", args.done)
+
+	if args.associatedIndexID != nil {
+		qs.SetInt("associatedIndexId", *args.associatedIndexID)
+	}
+
+	// Do not set an index of zero unless we're uploading a part
+	if args.uploadID != 0 && !args.multiPart && !args.done {
+		qs.SetInt("index", args.index)
+	}
+
+	return qs.Encode()
+}
+
+const ProgressUpdateInterval = time.Millisecond * 100
+
+// uploadFile performs an HTTP POST to the upload endpoint with the content from the given file.
+// This method will gzip the content before sending. If target is a non-nil pointer, it will be
+// assigned the value of the upload identifier present in the response body. If the events channel
+// is non-nil, progress of the upload will be sent to it on a timer. This function returns an error
+// as well as a boolean flag indicating if the function can be retried.
+func uploadFile(args requestArgs, file string, compressed bool, target *int, part, numParts int, events chan<- UploadProgressEvent, logger RequestLogger) (bool, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := f.Close(); closeErr != nil {
+			err = multierror.Append(err, closeErr)
+		}
+	}()
+
+	var r io.Reader = f
+
+	if events != nil {
+		info, err := os.Stat(file)
+		if err != nil {
+			return false, err
+		}
+
+		rateReader := newRateReader(f, info.Size())
+		progressPerFile := 1 / float64(numParts)
+
+		t := time.NewTicker(ProgressUpdateInterval)
+		defer t.Stop()
+
+		go func() {
+			for range t.C {
+				p := rateReader.Progress()
+				event := UploadProgressEvent{
+					NumParts:      numParts,
+					Part:          part + 1,
+					Progress:      p,
+					TotalProgress: float64(part)*progressPerFile + (p * progressPerFile),
+				}
+
+				select {
+				case events <- event:
+				default:
+				}
+			}
+		}()
+
+		r = rateReader
+	}
+
+	if !compressed {
+		r = Gzip(r)
+	}
+
+	return makeUploadRequest(args, r, target, logger)
+}
+
+// makeUploadRequest performs an HTTP POST to the upload endpoint. The query string of the request
+// is constructed from the given request args and the body of the request is the unmodified reader.
+// If target is a non-nil pointer, it will be assigned the value of the upload identifier present
+// in the response body. This function returns an error as well as a boolean flag indicating if the
+// function can be retried.
+func makeUploadRequest(args requestArgs, payload io.Reader, target *int, logger RequestLogger) (bool, error) {
+	url := args.baseURL
+	url.RawQuery = args.EncodeQuery()
+
+	req, err := http.NewRequest("POST", url.String(), payload)
+	if err != nil {
+		return false, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-ndjson+lsif")
+	if args.accessToken != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("token %s", args.accessToken))
+	}
+	for k, v := range args.additionalHeaders {
+		req.Header.Set(k, v)
+	}
+
+	started := time.Now()
+	if logger != nil {
+		logger.LogRequest(req)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return true, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if logger != nil {
+		logger.LogResponse(req, resp, body, time.Since(started))
+	}
+	if err != nil {
+		return true, err
+	}
+
+	if resp.StatusCode >= 300 {
+		if resp.StatusCode == http.StatusUnauthorized {
+			return false, ErrUnauthorized
+		}
+
+		// Do not retry client errors
+		err = fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		return resp.StatusCode >= 500, err
+	}
+
+	if target != nil {
+		var respPayload struct {
+			ID string `json:"id"`
+		}
+		if err := json.Unmarshal(body, &respPayload); err != nil {
+			return false, fmt.Errorf("unexpected response (%s)", err)
+		}
+
+		id, err := strconv.Atoi(respPayload.ID)
+		if err != nil {
+			return false, fmt.Errorf("unexpected response (%s)", err)
+		}
+
+		*target = id
+	}
+
+	return false, nil
+}
+
+// makeRetry returns a function that calls retry with the given max attempt and interval values.
+func makeRetry(n int, interval time.Duration) func(f func() (bool, error)) error {
+	return func(f func() (bool, error)) error {
+		return retry(f, n, interval)
+	}
+}
+
+// retry will re-invoke the given function until it returns a nil error value, the function returns
+// a non-retryable error (as indicated by its boolean return value), or until the maximum number of
+// retries have been attempted. The returned error will be the last error to occur.
+func retry(f func() (bool, error), n int, interval time.Duration) (err error) {
+	var retry bool
+	for i := n; i >= 0; i-- {
+		if retry, err = f(); err == nil || !retry {
+			break
+		}
+
+		time.Sleep(interval)
+	}
+
+	return err
+}
+
+// queryValues is a convenience wrapper around url.Values that adds
+// behaviors to set values of non-string types and optionally set
+// values that may be a zero-value.
+type queryValues struct {
+	values url.Values
+}
+
+// newQueryValues creates a new queryValues.
+func newQueryValues() queryValues {
+	return queryValues{values: url.Values{}}
+}
+
+// Set adds the given name/string-value pairing to the underlying values.
+func (qv queryValues) Set(name string, value string) {
+	qv.values[name] = []string{value}
+}
+
+// SetInt adds the given name/int-value pairing to the underlying values.
+func (qv queryValues) SetInt(name string, value int) {
+	qv.Set(name, strconv.FormatInt(int64(value), 10))
+}
+
+// SetOptionalString adds the given name/string-value pairing to the underlying values.
+// If the value is empty, the underlying values are not modified.
+func (qv queryValues) SetOptionalString(name string, value string) {
+	if value != "" {
+		qv.Set(name, value)
+	}
+}
+
+// SetOptionalInt adds the given name/int-value pairing to the underlying values.
+// If the value is zero, the underlying values are not modified.
+func (qv queryValues) SetOptionalInt(name string, value int) {
+	if value != 0 {
+		qv.SetInt(name, value)
+	}
+}
+
+// SetOptionalBool adds the given name/bool-value pairing to the underlying values.
+// If the value is false, the underlying values are not modified.
+func (qv queryValues) SetOptionalBool(name string, value bool) {
+	if value {
+		qv.Set(name, "true")
+	}
+}
+
+// Encode encodes the underlying values.
+func (qv queryValues) Encode() string {
+	return qv.values.Encode()
+}

enterprise/lib/codeintel/utils/upload_test.go

@@ -0,0 +1,155 @@
+package codeintelutils
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestUploadIndex(t *testing.T) {
+	var expectedPayload []byte
+	for i := 0; i < 500; i++ {
+		expectedPayload = append(expectedPayload, byte(i))
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		payload, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("unexpected error reading request body: %s", err)
+		}
+
+		gzipReader, err := gzip.NewReader(bytes.NewReader(payload))
+		if err != nil {
+			t.Fatalf("unexpected error creating gzip.Reader: %s", err)
+		}
+		decompressed, err := ioutil.ReadAll(gzipReader)
+		if err != nil {
+			t.Fatalf("unexpected error reading from gzip.Reader: %s", err)
+		}
+
+		if diff := cmp.Diff(expectedPayload, decompressed); diff != "" {
+			t.Errorf("unexpected request payload (-want +got):\n%s", diff)
+		}
+
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"id":"42"}`))
+	}))
+	defer ts.Close()
+
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("unexpected error creating temp file: %s", err)
+	}
+	defer func() { os.Remove(f.Name()) }()
+	_, _ = io.Copy(f, bytes.NewReader(expectedPayload))
+	_ = f.Close()
+
+	id, err := UploadIndex(UploadIndexOpts{
+		Endpoint:            ts.URL,
+		AccessToken:         "hunter2",
+		Repo:                "foo/bar",
+		Commit:              "deadbeef",
+		Root:                "proj/",
+		Indexer:             "lsif-go",
+		GitHubToken:         "ght",
+		File:                f.Name(),
+		MaxPayloadSizeBytes: 1000,
+	})
+	if err != nil {
+		t.Fatalf("unexpected error uploading index: %s", err)
+	}
+
+	if id != 42 {
+		t.Errorf("unexpected id. want=%d have=%d", 42, id)
+	}
+}
+
+func TestUploadIndexMultipart(t *testing.T) {
+	var expectedPayload []byte
+	for i := 0; i < 20000; i++ {
+		expectedPayload = append(expectedPayload, byte(i))
+	}
+
+	var m sync.Mutex
+	payloads := map[int][]byte{}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("multiPart") != "" {
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte(`{"id":"42"}`)) // graphql id is TFNJRlVwbG9hZDoiNDIi
+			return
+		}
+
+		if r.URL.Query().Get("index") != "" {
+			payload, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				t.Fatalf("unexpected error reading request body: %s", err)
+			}
+
+			index, _ := strconv.Atoi(r.URL.Query().Get("index"))
+			m.Lock()
+			payloads[index] = payload
+			m.Unlock()
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer ts.Close()
+
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("unexpected error creating temp file: %s", err)
+	}
+	defer func() { os.Remove(f.Name()) }()
+	_, _ = io.Copy(f, bytes.NewReader(expectedPayload))
+	_ = f.Close()
+
+	id, err := UploadIndex(UploadIndexOpts{
+		Endpoint:            ts.URL,
+		AccessToken:         "hunter2",
+		Repo:                "foo/bar",
+		Commit:              "deadbeef",
+		Root:                "proj/",
+		Indexer:             "lsif-go",
+		GitHubToken:         "ght",
+		File:                f.Name(),
+		MaxPayloadSizeBytes: 100,
+	})
+	if err != nil {
+		t.Fatalf("unexpected error uploading index: %s", err)
+	}
+
+	if id != 42 {
+		t.Errorf("unexpected id. want=%d have=%d", 42, id)
+	}
+
+	if len(payloads) != 5 {
+		t.Errorf("unexpected payloads size. want=%d have=%d", 5, len(payloads))
+	}
+
+	var allPayloads []byte
+	for i := 0; i < 5; i++ {
+		allPayloads = append(allPayloads, payloads[i]...)
+	}
+
+	gzipReader, err := gzip.NewReader(bytes.NewReader(allPayloads))
+	if err != nil {
+		t.Fatalf("unexpected error creating gzip.Reader: %s", err)
+	}
+	decompressed, err := ioutil.ReadAll(gzipReader)
+	if err != nil {
+		t.Fatalf("unexpected error reading from gzip.Reader: %s", err)
+	}
+	if diff := cmp.Diff(expectedPayload, decompressed); diff != "" {
+		t.Errorf("unexpected gzipped contents (-want +got):\n%s", diff)
+	}
+}

enterprise/lib/go.mod

@@ -4,5 +4,7 @@ go 1.15
 
 require (
 	github.com/google/go-cmp v0.5.4
+	github.com/hashicorp/go-multierror v1.1.0
 	github.com/json-iterator/go v1.1.10
+	github.com/pkg/errors v0.9.1
 )

enterprise/lib/go.sum

@@ -3,12 +3,18 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

go.mod

@@ -154,7 +154,6 @@ require (
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
 	github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect
 	github.com/sourcegraph/campaignutils v0.0.0-20201124155628-5d86cf20398d
-	github.com/sourcegraph/codeintelutils v0.0.0-20200824140252-1db3aed5cf58
 	github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81
 	github.com/sourcegraph/go-ctags v0.0.0-20201109224903-0e02e034fdb1
 	github.com/sourcegraph/go-diff v0.6.1

go.sum

@@ -1544,8 +1544,6 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d h1:yKm7XZV6j9
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/campaignutils v0.0.0-20201124155628-5d86cf20398d h1:BvdJF34dFf+M4anMVVGDaXDoAJbAonWV3SLILngzXds=
 github.com/sourcegraph/campaignutils v0.0.0-20201124155628-5d86cf20398d/go.mod h1:xm6i78Mk2t4DBLQDqEFc/3x6IPf7yYZCgbNaTQGhJHA=
-github.com/sourcegraph/codeintelutils v0.0.0-20200824140252-1db3aed5cf58 h1:Ps+U1xoZP+Zoph39YfRB6Q846ghO8pgrAgp0MiObvPs=
-github.com/sourcegraph/codeintelutils v0.0.0-20200824140252-1db3aed5cf58/go.mod h1:HplI8gRslTrTUUsSYwu28hSOderix7m5dHNca7xBzeo=
 github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81 h1:v4/JVxZSPWifxmICRqgXK7khThjw03RfdGhyeA2S4EQ=
 github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81/go.mod h1:xIvvI5FiHLxhv8prbzVpaMHaaGPFPFQSuTcxC91ryOo=
 github.com/sourcegraph/go-ctags v0.0.0-20201109224903-0e02e034fdb1 h1:di6PFtd0Fp/tvp4PddGk8yDrOwUpu2Y/UBZ39dgvLoc=
@@ -1575,8 +1573,6 @@ github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e h1:qpG
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152 h1:z/MpntplPaW6QW95pzcAR/72Z5TWDyDnSo0EOcyij9o=
 github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I=
-github.com/sourcegraph/zoekt v0.0.0-20210224103651-ed2848a6422c h1:iCwC2/0fZ2UJM/HhdT82dBwCaSgBllcp8PCoDC565qk=
-github.com/sourcegraph/zoekt v0.0.0-20210224103651-ed2848a6422c/go.mod h1:1Uv3ThqJ+VWQQS1qFIzufLA8jkQ6a+FDk3OFI6qPZVQ=
 github.com/sourcegraph/zoekt v0.0.0-20210225093519-da52e7c141aa h1:RiKzBMOLlWkKy7MSG1YZx6KTxIjMvHKd9eMR0egv34k=
 github.com/sourcegraph/zoekt v0.0.0-20210225093519-da52e7c141aa/go.mod h1:1Uv3ThqJ+VWQQS1qFIzufLA8jkQ6a+FDk3OFI6qPZVQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -1638,8 +1634,6 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
 github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
-github.com/throttled/throttled v1.0.0 h1:GQ7a1ilkYMXnZ6V0EG7RiU3qHk1k/kpXO/aIQQXPfEQ=
-github.com/throttled/throttled v2.2.5+incompatible h1:65UB52X0qNTYiT0Sohp8qLYVFwZQPDw85uSa65OljjQ=
 github.com/throttled/throttled/v2 v2.7.1 h1:FnBysDX4Sok55bvfDMI0l2Y71V1vM2wi7O79OW7fNtw=
 github.com/throttled/throttled/v2 v2.7.1/go.mod h1:fuOeyK9fmnA+LQnsBbfT/mmPHjmkdogRBQxaD8YsgZ8=
 github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=

third-party-licenses/ThirdPartyLicenses.csv

@@ -320,7 +320,6 @@ Go,github.com/sirupsen/logrus,v1.6.0,MIT,"",Approved
 Go,github.com/sourcegraph/alertmanager,v0.21.1-0.20200727091526-3e856a90b534,Apache 2.0,"",Approved
 Go,github.com/sourcegraph/annotate,v0.0.0-20160123013949-f4cad6c6324d,New BSD,"",Approved
 Go,github.com/sourcegraph/campaignutils,v0.0.0-20201124155628-5d86cf20398d,Apache 2.0,"",Approved
-Go,github.com/sourcegraph/codeintelutils,v0.0.0-20200824140252-1db3aed5cf58,MIT,"",Approved
 Go,github.com/sourcegraph/ctxvfs,v0.0.0-20180418081416-2b65f1b1ea81,MIT,"",Approved
 Go,github.com/sourcegraph/go-ctags,v0.0.0-20201109224903-0e02e034fdb1,Apache 2.0,"",Approved
 Go,github.com/sourcegraph/go-diff,v0.6.1,MIT,"",Approved