artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 15:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(log): remove use of log15 in ghe-feeder (#64131)

Browse Source 
update ghe-feeder to use sourcegraph/log. As a consequence ghe-feeder no
longer has the logFile flag - instead people should just redirect the
output to a file with >> / >

Closes DINF-153

<!-- PR description tips:
https://www.notion.so/sourcegraph/Write-a-good-pull-request-description-610a7fd3e613496eb76f450db5a49b6e
-->

## Test plan
CI
<!-- REQUIRED; info at
https://docs-legacy.sourcegraph.com/dev/background-information/testing_principles
-->

## Changelog

<!-- OPTIONAL; info at
https://www.notion.so/sourcegraph/Writing-a-changelog-entry-dd997f411d524caabf0d8d38a24a878c
-->
This commit is contained in:
William Bezuidenhout 2024-07-30 11:31:38 +02:00 committed by GitHub
parent 963527ccd9
commit 16d1ab7586
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 39 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/cmd/ghe-feeder/BUILD.bazel
View File

@ -16,12 +16,12 @@ go_library(
"//internal/ratelimit",
"//lib/errors",
"@com_github_google_go_github_v55//github",
"@com_github_inconshreveable_log15//:log15",
"@com_github_mattn_go_sqlite3//:go-sqlite3",
"@com_github_prometheus_client_golang//prometheus",
"@com_github_prometheus_client_golang//prometheus/promauto",
"@com_github_prometheus_client_golang//prometheus/promhttp",
"@com_github_schollz_progressbar_v3//:progressbar",
"@com_github_sourcegraph_log//:log",
"@org_golang_x_oauth2//:oauth2",
"@org_golang_x_time//rate",
],

36
internal/cmd/ghe-feeder/main.go
View File

@ -4,7 +4,6 @@ import (
"context"
"flag"
"fmt"
"log" //nolint:logging // TODO move all logging to sourcegraph/log
"math"
"net/http"
"net/url"
@ -14,13 +13,13 @@ import (
"sync"
"time"
"github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/schollz/progressbar/v3"
"golang.org/x/time/rate"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
)
@ -58,7 +57,6 @@ func main() {
scratchDir := flag.String("scratchDir", "", "scratch dir where to temporarily clone repositories")
limitPump := flag.Int64("limit", math.MaxInt64, "limit processing to this many repos (for debugging)")
skipNumLines := flag.Int64("skip", 0, "skip this many lines from input")
logFilepath := flag.String("logfile", "feeder.log", "path to a log file")
apiCallsPerSec := flag.Float64("apiCallsPerSec", 100.0, "how many API calls per sec to destination GHE")
numSimultaneousPushes := flag.Int("numSimultaneousPushes", 10, "number of simultaneous GHE pushes")
cloneRepoTimeout := flag.Duration("cloneRepoTimeout", time.Minute*3, "how long to wait for a repo to clone")
@ -70,11 +68,9 @@ func main() {
flag.Parse()
logHandler, err := log15.FileHandler(*logFilepath, log15.LogfmtFormat())
if err != nil {
log.Fatal(err)
}
log15.Root().SetHandler(logHandler)
liblog := log.Init(log.Resource{Name: "ghe-feeder"})
defer liblog.Sync()
logger := log.Scoped("main")
if *help || len(*baseURL) == 0 || len(*token) == 0 || len(*admin) == 0 {
flag.PrintDefaults()
@ -88,7 +84,7 @@ func main() {
if len(*scratchDir) == 0 {
d, err := os.MkdirTemp("", "ghe-feeder")
if err != nil {
log15.Error("failed to create scratch dir", "error", err)
logger.Error("failed to create scratch dir", log.Error(err))
os.Exit(1)
}
*scratchDir = d
@ -96,7 +92,7 @@ func main() {
u, err := url.Parse(*baseURL)
if err != nil {
log15.Error("failed to parse base URL", "baseURL", *baseURL, "error", err)
logger.Error("failed to parse base URL", log.String("baseURL", *baseURL), log.Error(err))
os.Exit(1)
}
host := u.Host
@ -104,20 +100,20 @@ func main() {
ctx := context.Background()
gheClient, err := newGHEClient(ctx, *baseURL, *uploadURL, *token)
if err != nil {
log15.Error("failed to create GHE client", "error", err)
logger.Error("failed to create GHE client", log.Error(err))
os.Exit(1)
}
fdr, err := newFeederDB(*progressFilepath)
if err != nil {
log15.Error("failed to create sqlite DB", "path", *progressFilepath, "error", err)
logger.Error("failed to create sqlite DB", log.String("path", *progressFilepath), log.Error(err))
os.Exit(1)
}
spinner := progressbar.Default(-1, "calculating work")
numLines, err := numLinesTotal(*skipNumLines)
if err != nil {
log15.Error("failed to calculate outstanding work", "error", err)
logger.Error("failed to calculate outstanding work", log.Error(err))
os.Exit(1)
}
_ = spinner.Finish()
@ -127,7 +123,7 @@ func main() {
}
if numLines == 0 {
log15.Info("no work remaining in input")
logger.Info("no work remaining in input")
fmt.Println("no work remaining in input, exiting")
os.Exit(0)
}
@ -141,7 +137,7 @@ func main() {
remaining: numLines,
pipe: work,
fdr: fdr,
logger: log15.New("source", "producer"),
logger: logger.Scoped("producer"),
bar: bar,
skipNumLines: *skipNumLines,
}
@ -182,8 +178,7 @@ func main() {
wkrScratchDir := filepath.Join(*scratchDir, name)
err := os.MkdirAll(wkrScratchDir, 0777)
if err != nil {
log15.Error("failed to create worker scratch dir", "scratchDir", *scratchDir, "error", err)
os.Exit(1)
logger.Fatal("failed to create worker scratch dir", log.String("scratchDir", *scratchDir), log.Error(err))
}
wkr := &worker{
@ -195,7 +190,7 @@ func main() {
wg: &wg,
bar: bar,
fdr: fdr,
logger: log15.New("source", name),
logger: logger.Scoped(name),
rateLimiter: rateLimiter,
admin: *admin,
token: *token,
@ -216,8 +211,7 @@ func main() {
err = prdc.pump(ctx)
if err != nil {
log15.Error("pump failed", "error", err)
os.Exit(1)
logger.Fatal("pump failed", log.Error(err))
}
close(work)
wg.Wait()
@ -226,7 +220,7 @@ func main() {
s := stats(wkrs, prdc)
fmt.Println(s)
log15.Info(s)
logger.Info(s)
}
func stats(wkrs []*worker, prdc *producer) string {

8
internal/cmd/ghe-feeder/progress.go
View File

@ -4,7 +4,6 @@ import (
"database/sql"
"sync"
"github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log
_ "github.com/mattn/go-sqlite3"
"github.com/sourcegraph/sourcegraph/internal/database/dbutil"
@ -19,8 +18,6 @@ type feederDB struct {
path string
// the opened DB
db *sql.DB
// logger for this feeder DB
logger log15.Logger
}
// newFeederDB creates or opens the DB, creating the two tables if necessary
@ -48,9 +45,8 @@ func newFeederDB(path string) (*feederDB, error) {
}
return &feederDB{
path: path,
db: db,
logger: log15.New("source", "feederDB"),
path: path,
db: db,
}, nil
}

6
internal/cmd/ghe-feeder/pump.go
View File

@ -9,9 +9,9 @@ import (
"path/filepath"
"strings"
"github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log
"github.com/prometheus/client_golang/prometheus"
"github.com/schollz/progressbar/v3"
"github.com/sourcegraph/log"
)
// extractOwnerRepoFromCSVLine extracts the owner and repo from a line that comes from a CSV file that a GHE instance
@ -44,7 +44,7 @@ type producer struct {
// how many we have already processed
numAlreadyDone int64
// logger for the pump
logger log15.Logger
logger log.Logger
// terminal UI progress bar
bar *progressbar.ProgressBar
// skips this many lines from the input before starting to feed into the pipe
@ -89,7 +89,7 @@ func (prdc *producer) pumpFile(ctx context.Context, path string) error {
reposSucceededCounter.Inc()
remainingWorkGauge.Add(-1.0)
prdc.remaining--
prdc.logger.Debug("repo already done in previous run", "owner/repo", line)
prdc.logger.Debug("repo already done in previous run", log.String("owner/repo", line))
continue
}
select {

36
internal/cmd/ghe-feeder/worker.go
View File

@ -12,11 +12,11 @@ import (
"time"
"github.com/google/go-github/v55/github"
"github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log
"github.com/prometheus/client_golang/prometheus"
"github.com/schollz/progressbar/v3"
"golang.org/x/oauth2"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -98,7 +98,7 @@ type worker struct {
currentMaxRepos int
// logger has worker name inprinted
logger log15.Logger
logger log.Logger
// rate limiter for the GHE API calls
rateLimiter *ratelimit.InstrumentedLimiter
@ -123,18 +123,18 @@ func (wkr *worker) run(ctx context.Context) {
wkr.currentOrg, wkr.currentMaxRepos = randomOrgNameAndSize()
}
wkr.logger.Debug("switching to org", "org", wkr.currentOrg)
wkr.logger.Debug("switching to org", log.String("org", wkr.currentOrg))
// declare the first org to start the worker processing
err := wkr.addGHEOrg(ctx)
if err != nil {
wkr.logger.Error("failed to create org", "org", wkr.currentOrg, "error", err)
wkr.logger.Error("failed to create org", log.String("org", wkr.currentOrg), log.Error(err))
// add it to default org then
wkr.currentOrg = ""
} else {
err = wkr.fdr.declareOrg(wkr.currentOrg)
if err != nil {
wkr.logger.Error("failed to declare org", "org", wkr.currentOrg, "error", err)
wkr.logger.Error("failed to declare org", log.String("org", wkr.currentOrg), log.Error(err))
}
}
@ -147,7 +147,7 @@ func (wkr *worker) run(ctx context.Context) {
xs := strings.Split(line, "/")
if len(xs) != 2 {
wkr.logger.Error("failed tos split line", "line", line)
wkr.logger.Error("failed tos split line", log.String("line", line))
continue
}
owner, repo := xs[0], xs[1]
@ -172,23 +172,23 @@ func (wkr *worker) run(ctx context.Context) {
err = wkr.fdr.succeeded(line, wkr.currentOrg)
if err != nil {
wkr.logger.Error("failed to mark succeeded repo", "ownerRepo", line, "error", err)
wkr.logger.Error("failed to mark succeeded repo", log.String("ownerRepo", line), log.Error(err))
}
// switch to a new org
if wkr.currentNumRepos >= wkr.currentMaxRepos {
wkr.currentOrg, wkr.currentMaxRepos = randomOrgNameAndSize()
wkr.currentNumRepos = 0
wkr.logger.Debug("switching to org", "org", wkr.currentOrg)
wkr.logger.Debug("switching to org", log.String("org", wkr.currentOrg))
err := wkr.addGHEOrg(ctx)
if err != nil {
wkr.logger.Error("failed to create org", "org", wkr.currentOrg, "error", err)
wkr.logger.Error("failed to create org", log.String("org", wkr.currentOrg), log.Error(err))
// add it to default org then
wkr.currentOrg = ""
} else {
err = wkr.fdr.declareOrg(wkr.currentOrg)
if err != nil {
wkr.logger.Error("failed to declare org", "org", wkr.currentOrg, "error", err)
wkr.logger.Error("failed to declare org", log.String("org", wkr.currentOrg), log.Error(err))
}
}
}
@ -198,7 +198,7 @@ func (wkr *worker) run(ctx context.Context) {
// clean up clone on disk
err = os.RemoveAll(ownerDir)
if err != nil {
wkr.logger.Error("failed to clean up cloned repo", "ownerRepo", line, "error", err, "ownerDir", ownerDir)
wkr.logger.Error("failed to clean up cloned repo", log.String("ownerRepo", line), log.String("ownerDir", ownerDir), log.Error(err))
}
}
}
@ -207,19 +207,19 @@ func (wkr *worker) run(ctx context.Context) {
func (wkr *worker) process(ctx context.Context, owner, repo string) error {
err := wkr.cloneRepo(ctx, owner, repo)
if err != nil {
wkr.logger.Error("failed to clone repo", "owner", owner, "repo", repo, "error", err)
wkr.logger.Error("failed to clone repo", log.String("owner", owner), log.String("repo", repo), log.Error(err))
return &feederError{"clone", err}
}
gheRepo, err := wkr.addGHERepo(ctx, owner, repo)
if err != nil {
wkr.logger.Error("failed to create GHE repo", "owner", owner, "repo", repo, "error", err)
wkr.logger.Error("failed to create GHE repo", log.String("owner", owner), log.String("repo", repo), log.Error(err))
return &feederError{"api", err}
}
err = wkr.addRemote(ctx, gheRepo, owner, repo)
if err != nil {
wkr.logger.Error("failed to add GHE as a remote in cloned repo", "owner", owner, "repo", repo, "error", err)
wkr.logger.Error("failed to add GHE as a remote in cloned repo", log.String("owner", owner), log.String("repo", repo), log.Error(err))
return &feederError{"api", err}
}
@ -228,7 +228,7 @@ func (wkr *worker) process(ctx context.Context, owner, repo string) error {
if err == nil {
return nil
}
wkr.logger.Error("failed to push cloned repo to GHE", "attempt", attempt+1, "owner", owner, "repo", repo, "error", err)
wkr.logger.Error("failed to push cloned repo to GHE", log.Int("attempt", attempt+1), log.String("owner", owner), log.String("repo", repo), log.Error(err))
}
if ctx.Err() != nil {
@ -248,7 +248,7 @@ func (wkr *worker) cloneRepo(ctx context.Context, owner, repo string) error {
ownerDir := filepath.Join(wkr.scratchDir, owner)
err := os.MkdirAll(ownerDir, 0o777)
if err != nil {
wkr.logger.Error("failed to create owner dir", "ownerDir", ownerDir, "error", err)
wkr.logger.Error("failed to create owner dir", log.String("ownerDir", ownerDir), log.Error(err))
return err
}
@ -302,7 +302,7 @@ func (wkr *worker) pushToGHE(ctx context.Context, owner, repo string) error {
func (wkr *worker) addGHEOrg(ctx context.Context) error {
err := wkr.rateLimiter.Wait(ctx)
if err != nil {
wkr.logger.Error("failed to get a request spot from rate limiter", "error", err)
wkr.logger.Error("failed to get a request spot from rate limiter", log.Error(err))
return err
}
@ -321,7 +321,7 @@ func (wkr *worker) addGHEOrg(ctx context.Context) error {
func (wkr *worker) addGHERepo(ctx context.Context, owner, repo string) (*github.Repository, error) {
err := wkr.rateLimiter.Wait(ctx)
if err != nil {
wkr.logger.Error("failed to get a request spot from rate limiter", "error", err)
wkr.logger.Error("failed to get a request spot from rate limiter", log.Error(err))
return nil, err
}