From 16d1ab7586d58b2798c42477c1b6181eb13df52e Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Tue, 30 Jul 2024 11:31:38 +0200 Subject: [PATCH] chore(log): remove use of log15 in ghe-feeder (#64131) update ghe-feeder to use sourcegraph/log. As a consequence ghe-feeder no longer has the logFile flag - instead people should just redirect the output to a file with >> / > Closes DINF-153 ## Test plan CI ## Changelog --- internal/cmd/ghe-feeder/BUILD.bazel | 2 +- internal/cmd/ghe-feeder/main.go | 36 ++++++++++++----------------- internal/cmd/ghe-feeder/progress.go | 8 ++----- internal/cmd/ghe-feeder/pump.go | 6 ++--- internal/cmd/ghe-feeder/worker.go | 36 ++++++++++++++--------------- 5 files changed, 39 insertions(+), 49 deletions(-) diff --git a/internal/cmd/ghe-feeder/BUILD.bazel b/internal/cmd/ghe-feeder/BUILD.bazel index 765f5e22479..4240e73a54a 100644 --- a/internal/cmd/ghe-feeder/BUILD.bazel +++ b/internal/cmd/ghe-feeder/BUILD.bazel @@ -16,12 +16,12 @@ go_library( "//internal/ratelimit", "//lib/errors", "@com_github_google_go_github_v55//github", - "@com_github_inconshreveable_log15//:log15", "@com_github_mattn_go_sqlite3//:go-sqlite3", "@com_github_prometheus_client_golang//prometheus", "@com_github_prometheus_client_golang//prometheus/promauto", "@com_github_prometheus_client_golang//prometheus/promhttp", "@com_github_schollz_progressbar_v3//:progressbar", + "@com_github_sourcegraph_log//:log", "@org_golang_x_oauth2//:oauth2", "@org_golang_x_time//rate", ], diff --git a/internal/cmd/ghe-feeder/main.go b/internal/cmd/ghe-feeder/main.go index a7271404041..8f911806aac 100644 --- a/internal/cmd/ghe-feeder/main.go +++ b/internal/cmd/ghe-feeder/main.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "log" //nolint:logging // TODO move all logging to sourcegraph/log "math" "net/http" "net/url" @@ -14,13 +13,13 @@ import ( "sync" "time" - "github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/schollz/progressbar/v3" "golang.org/x/time/rate" + "github.com/sourcegraph/log" "github.com/sourcegraph/sourcegraph/internal/ratelimit" ) @@ -58,7 +57,6 @@ func main() { scratchDir := flag.String("scratchDir", "", "scratch dir where to temporarily clone repositories") limitPump := flag.Int64("limit", math.MaxInt64, "limit processing to this many repos (for debugging)") skipNumLines := flag.Int64("skip", 0, "skip this many lines from input") - logFilepath := flag.String("logfile", "feeder.log", "path to a log file") apiCallsPerSec := flag.Float64("apiCallsPerSec", 100.0, "how many API calls per sec to destination GHE") numSimultaneousPushes := flag.Int("numSimultaneousPushes", 10, "number of simultaneous GHE pushes") cloneRepoTimeout := flag.Duration("cloneRepoTimeout", time.Minute*3, "how long to wait for a repo to clone") @@ -70,11 +68,9 @@ func main() { flag.Parse() - logHandler, err := log15.FileHandler(*logFilepath, log15.LogfmtFormat()) - if err != nil { - log.Fatal(err) - } - log15.Root().SetHandler(logHandler) + liblog := log.Init(log.Resource{Name: "ghe-feeder"}) + defer liblog.Sync() + logger := log.Scoped("main") if *help || len(*baseURL) == 0 || len(*token) == 0 || len(*admin) == 0 { flag.PrintDefaults() @@ -88,7 +84,7 @@ func main() { if len(*scratchDir) == 0 { d, err := os.MkdirTemp("", "ghe-feeder") if err != nil { - log15.Error("failed to create scratch dir", "error", err) + logger.Error("failed to create scratch dir", log.Error(err)) os.Exit(1) } *scratchDir = d @@ -96,7 +92,7 @@ func main() { u, err := url.Parse(*baseURL) if err != nil { - log15.Error("failed to parse base URL", "baseURL", *baseURL, "error", err) + logger.Error("failed to parse base URL", log.String("baseURL", *baseURL), log.Error(err)) os.Exit(1) } host := u.Host @@ -104,20 +100,20 @@ func main() { ctx := context.Background() gheClient, err := newGHEClient(ctx, *baseURL, *uploadURL, *token) if err != nil { - log15.Error("failed to create GHE client", "error", err) + logger.Error("failed to create GHE client", log.Error(err)) os.Exit(1) } fdr, err := newFeederDB(*progressFilepath) if err != nil { - log15.Error("failed to create sqlite DB", "path", *progressFilepath, "error", err) + logger.Error("failed to create sqlite DB", log.String("path", *progressFilepath), log.Error(err)) os.Exit(1) } spinner := progressbar.Default(-1, "calculating work") numLines, err := numLinesTotal(*skipNumLines) if err != nil { - log15.Error("failed to calculate outstanding work", "error", err) + logger.Error("failed to calculate outstanding work", log.Error(err)) os.Exit(1) } _ = spinner.Finish() @@ -127,7 +123,7 @@ func main() { } if numLines == 0 { - log15.Info("no work remaining in input") + logger.Info("no work remaining in input") fmt.Println("no work remaining in input, exiting") os.Exit(0) } @@ -141,7 +137,7 @@ func main() { remaining: numLines, pipe: work, fdr: fdr, - logger: log15.New("source", "producer"), + logger: logger.Scoped("producer"), bar: bar, skipNumLines: *skipNumLines, } @@ -182,8 +178,7 @@ func main() { wkrScratchDir := filepath.Join(*scratchDir, name) err := os.MkdirAll(wkrScratchDir, 0777) if err != nil { - log15.Error("failed to create worker scratch dir", "scratchDir", *scratchDir, "error", err) - os.Exit(1) + logger.Fatal("failed to create worker scratch dir", log.String("scratchDir", *scratchDir), log.Error(err)) } wkr := &worker{ @@ -195,7 +190,7 @@ func main() { wg: &wg, bar: bar, fdr: fdr, - logger: log15.New("source", name), + logger: logger.Scoped(name), rateLimiter: rateLimiter, admin: *admin, token: *token, @@ -216,8 +211,7 @@ func main() { err = prdc.pump(ctx) if err != nil { - log15.Error("pump failed", "error", err) - os.Exit(1) + logger.Fatal("pump failed", log.Error(err)) } close(work) wg.Wait() @@ -226,7 +220,7 @@ func main() { s := stats(wkrs, prdc) fmt.Println(s) - log15.Info(s) + logger.Info(s) } func stats(wkrs []*worker, prdc *producer) string { diff --git a/internal/cmd/ghe-feeder/progress.go b/internal/cmd/ghe-feeder/progress.go index 3f5bc8ff037..be89f8d8b87 100644 --- a/internal/cmd/ghe-feeder/progress.go +++ b/internal/cmd/ghe-feeder/progress.go @@ -4,7 +4,6 @@ import ( "database/sql" "sync" - "github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log _ "github.com/mattn/go-sqlite3" "github.com/sourcegraph/sourcegraph/internal/database/dbutil" @@ -19,8 +18,6 @@ type feederDB struct { path string // the opened DB db *sql.DB - // logger for this feeder DB - logger log15.Logger } // newFeederDB creates or opens the DB, creating the two tables if necessary @@ -48,9 +45,8 @@ func newFeederDB(path string) (*feederDB, error) { } return &feederDB{ - path: path, - db: db, - logger: log15.New("source", "feederDB"), + path: path, + db: db, }, nil } diff --git a/internal/cmd/ghe-feeder/pump.go b/internal/cmd/ghe-feeder/pump.go index 4351ae8bf37..a2d3c7758f1 100644 --- a/internal/cmd/ghe-feeder/pump.go +++ b/internal/cmd/ghe-feeder/pump.go @@ -9,9 +9,9 @@ import ( "path/filepath" "strings" - "github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log "github.com/prometheus/client_golang/prometheus" "github.com/schollz/progressbar/v3" + "github.com/sourcegraph/log" ) // extractOwnerRepoFromCSVLine extracts the owner and repo from a line that comes from a CSV file that a GHE instance @@ -44,7 +44,7 @@ type producer struct { // how many we have already processed numAlreadyDone int64 // logger for the pump - logger log15.Logger + logger log.Logger // terminal UI progress bar bar *progressbar.ProgressBar // skips this many lines from the input before starting to feed into the pipe @@ -89,7 +89,7 @@ func (prdc *producer) pumpFile(ctx context.Context, path string) error { reposSucceededCounter.Inc() remainingWorkGauge.Add(-1.0) prdc.remaining-- - prdc.logger.Debug("repo already done in previous run", "owner/repo", line) + prdc.logger.Debug("repo already done in previous run", log.String("owner/repo", line)) continue } select { diff --git a/internal/cmd/ghe-feeder/worker.go b/internal/cmd/ghe-feeder/worker.go index cfec78b7d6e..f4482471fe7 100644 --- a/internal/cmd/ghe-feeder/worker.go +++ b/internal/cmd/ghe-feeder/worker.go @@ -12,11 +12,11 @@ import ( "time" "github.com/google/go-github/v55/github" - "github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log "github.com/prometheus/client_golang/prometheus" "github.com/schollz/progressbar/v3" "golang.org/x/oauth2" + "github.com/sourcegraph/log" "github.com/sourcegraph/sourcegraph/internal/ratelimit" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -98,7 +98,7 @@ type worker struct { currentMaxRepos int // logger has worker name inprinted - logger log15.Logger + logger log.Logger // rate limiter for the GHE API calls rateLimiter *ratelimit.InstrumentedLimiter @@ -123,18 +123,18 @@ func (wkr *worker) run(ctx context.Context) { wkr.currentOrg, wkr.currentMaxRepos = randomOrgNameAndSize() } - wkr.logger.Debug("switching to org", "org", wkr.currentOrg) + wkr.logger.Debug("switching to org", log.String("org", wkr.currentOrg)) // declare the first org to start the worker processing err := wkr.addGHEOrg(ctx) if err != nil { - wkr.logger.Error("failed to create org", "org", wkr.currentOrg, "error", err) + wkr.logger.Error("failed to create org", log.String("org", wkr.currentOrg), log.Error(err)) // add it to default org then wkr.currentOrg = "" } else { err = wkr.fdr.declareOrg(wkr.currentOrg) if err != nil { - wkr.logger.Error("failed to declare org", "org", wkr.currentOrg, "error", err) + wkr.logger.Error("failed to declare org", log.String("org", wkr.currentOrg), log.Error(err)) } } @@ -147,7 +147,7 @@ func (wkr *worker) run(ctx context.Context) { xs := strings.Split(line, "/") if len(xs) != 2 { - wkr.logger.Error("failed tos split line", "line", line) + wkr.logger.Error("failed tos split line", log.String("line", line)) continue } owner, repo := xs[0], xs[1] @@ -172,23 +172,23 @@ func (wkr *worker) run(ctx context.Context) { err = wkr.fdr.succeeded(line, wkr.currentOrg) if err != nil { - wkr.logger.Error("failed to mark succeeded repo", "ownerRepo", line, "error", err) + wkr.logger.Error("failed to mark succeeded repo", log.String("ownerRepo", line), log.Error(err)) } // switch to a new org if wkr.currentNumRepos >= wkr.currentMaxRepos { wkr.currentOrg, wkr.currentMaxRepos = randomOrgNameAndSize() wkr.currentNumRepos = 0 - wkr.logger.Debug("switching to org", "org", wkr.currentOrg) + wkr.logger.Debug("switching to org", log.String("org", wkr.currentOrg)) err := wkr.addGHEOrg(ctx) if err != nil { - wkr.logger.Error("failed to create org", "org", wkr.currentOrg, "error", err) + wkr.logger.Error("failed to create org", log.String("org", wkr.currentOrg), log.Error(err)) // add it to default org then wkr.currentOrg = "" } else { err = wkr.fdr.declareOrg(wkr.currentOrg) if err != nil { - wkr.logger.Error("failed to declare org", "org", wkr.currentOrg, "error", err) + wkr.logger.Error("failed to declare org", log.String("org", wkr.currentOrg), log.Error(err)) } } } @@ -198,7 +198,7 @@ func (wkr *worker) run(ctx context.Context) { // clean up clone on disk err = os.RemoveAll(ownerDir) if err != nil { - wkr.logger.Error("failed to clean up cloned repo", "ownerRepo", line, "error", err, "ownerDir", ownerDir) + wkr.logger.Error("failed to clean up cloned repo", log.String("ownerRepo", line), log.String("ownerDir", ownerDir), log.Error(err)) } } } @@ -207,19 +207,19 @@ func (wkr *worker) run(ctx context.Context) { func (wkr *worker) process(ctx context.Context, owner, repo string) error { err := wkr.cloneRepo(ctx, owner, repo) if err != nil { - wkr.logger.Error("failed to clone repo", "owner", owner, "repo", repo, "error", err) + wkr.logger.Error("failed to clone repo", log.String("owner", owner), log.String("repo", repo), log.Error(err)) return &feederError{"clone", err} } gheRepo, err := wkr.addGHERepo(ctx, owner, repo) if err != nil { - wkr.logger.Error("failed to create GHE repo", "owner", owner, "repo", repo, "error", err) + wkr.logger.Error("failed to create GHE repo", log.String("owner", owner), log.String("repo", repo), log.Error(err)) return &feederError{"api", err} } err = wkr.addRemote(ctx, gheRepo, owner, repo) if err != nil { - wkr.logger.Error("failed to add GHE as a remote in cloned repo", "owner", owner, "repo", repo, "error", err) + wkr.logger.Error("failed to add GHE as a remote in cloned repo", log.String("owner", owner), log.String("repo", repo), log.Error(err)) return &feederError{"api", err} } @@ -228,7 +228,7 @@ func (wkr *worker) process(ctx context.Context, owner, repo string) error { if err == nil { return nil } - wkr.logger.Error("failed to push cloned repo to GHE", "attempt", attempt+1, "owner", owner, "repo", repo, "error", err) + wkr.logger.Error("failed to push cloned repo to GHE", log.Int("attempt", attempt+1), log.String("owner", owner), log.String("repo", repo), log.Error(err)) } if ctx.Err() != nil { @@ -248,7 +248,7 @@ func (wkr *worker) cloneRepo(ctx context.Context, owner, repo string) error { ownerDir := filepath.Join(wkr.scratchDir, owner) err := os.MkdirAll(ownerDir, 0o777) if err != nil { - wkr.logger.Error("failed to create owner dir", "ownerDir", ownerDir, "error", err) + wkr.logger.Error("failed to create owner dir", log.String("ownerDir", ownerDir), log.Error(err)) return err } @@ -302,7 +302,7 @@ func (wkr *worker) pushToGHE(ctx context.Context, owner, repo string) error { func (wkr *worker) addGHEOrg(ctx context.Context) error { err := wkr.rateLimiter.Wait(ctx) if err != nil { - wkr.logger.Error("failed to get a request spot from rate limiter", "error", err) + wkr.logger.Error("failed to get a request spot from rate limiter", log.Error(err)) return err } @@ -321,7 +321,7 @@ func (wkr *worker) addGHEOrg(ctx context.Context) error { func (wkr *worker) addGHERepo(ctx context.Context, owner, repo string) (*github.Repository, error) { err := wkr.rateLimiter.Wait(ctx) if err != nil { - wkr.logger.Error("failed to get a request spot from rate limiter", "error", err) + wkr.logger.Error("failed to get a request spot from rate limiter", log.Error(err)) return nil, err }