artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 12:51:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

gitserver: Merge enterprise and non-enterprise cmd (#56214)

Browse Source 
Just some cleanup after the relicensing, gitserver is now in a single package, and no enterprise specific version exists anymore.
This commit is contained in:
Erik Seliger 2023-08-25 13:25:07 +02:00 committed by GitHub
parent 36a01557ca
commit 92f4e865ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 26 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/gitserver/BUILD.bazel
View File

@ -1,7 +1,8 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball")
load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("@container_structure_test//:defs.bzl", "container_structure_test")
load("//dev:oci_defs.bzl", "image_repository")
go_library(
name = "gitserver_lib",
@ -36,9 +37,9 @@ pkg_tar(
"p4-fusion-wrapper-detect-kill.sh",
"process-stats-watcher.sh",
],
package_dir = "/usr/local/bin",
remap_paths = {
"/p4-fusion-wrapper-detect-kill.sh": "/p4-fusion",
"/p4-fusion-wrapper-detect-kill.sh": "/usr/local/bin/p4-fusion",
"/process-stats-watcher.sh": "/usr/local/bin/process-stats-watcher.sh",
},
visibility = ["//visibility:public"],
)
@ -76,3 +77,9 @@ container_structure_test(
"requires-network",
],
)
oci_push(
name = "candidate_push",
image = ":image",
repository = image_repository("gitserver"),
)

1
cmd/gitserver/p4-fusion-wrapper-detect-kill.sh
View File

@ -2,6 +2,7 @@
# shellcheck disable=SC2064,SC2207
# create a file to hold the output of p4-fusion
# TODO: consider recording/storing/capturing the file for logs display in the UI if there's a problem
fusionout=$(mktemp || mktemp -t fusionout_XXXXXXXX)
# create a pipe to use for capturing output of p4-fusion

1
cmd/gitserver/shared/BUILD.bazel
View File

@ -18,6 +18,7 @@ go_library(
"//internal/actor",
"//internal/api",
"//internal/authz",
"//internal/authz/subrepoperms",
"//internal/codeintel/dependencies",
"//internal/collections",
"//internal/conf",

2
cmd/gitserver/shared/service.go
View File

@ -23,7 +23,7 @@ func (svc) Configure() (env.Config, []debugserver.Endpoint) {
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return Main(ctx, observationCtx, ready, config.(*Config), nil)
return Main(ctx, observationCtx, ready, config.(*Config))
}
var Service service.Service = svc{}

11
cmd/gitserver/shared/shared.go
View File

@ -26,6 +26,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/authz/subrepoperms"
"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies"
"github.com/sourcegraph/sourcegraph/internal/collections"
"github.com/sourcegraph/sourcegraph/internal/conf"
@ -60,9 +61,7 @@ import (
"github.com/sourcegraph/sourcegraph/schema"
)
type EnterpriseInit func(db database.DB, keyring keyring.Ring)
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config *Config, enterpriseInit EnterpriseInit) error {
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config *Config) error {
logger := observationCtx.Logger
// Load and validate configuration.
@ -112,9 +111,9 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
return errors.Wrap(err, "initializing keyring")
}
// Possibly run enterprise hooks.
if enterpriseInit != nil {
enterpriseInit(db, keyring.Default())
authz.DefaultSubRepoPermsChecker, err = subrepoperms.NewSubRepoPermsClient(db.SubRepoPerms())
if err != nil {
return errors.Wrap(err, "failed to create sub-repo client")
}
// Setup our server megastruct.

1
dev/check/go-dbconn-import.sh
View File

@ -17,7 +17,6 @@ allowed_prefix=(
github.com/sourcegraph/sourcegraph/cmd/repo-updater
github.com/sourcegraph/sourcegraph/cmd/migrator
github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend
github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver
github.com/sourcegraph/sourcegraph/enterprise/cmd/worker
github.com/sourcegraph/sourcegraph/enterprise/cmd/migrator
github.com/sourcegraph/sourcegraph/cmd/precise-code-intel-worker

4
doc/dev/background-information/sg/index.md
View File

@ -193,8 +193,8 @@ commands:
install: |
config=""
[[ $(uname) == "Darwin" ]] && config="--config darwin-docker"
bazel build //enterprise/cmd/gitserver:image_tarball ${config} && \
docker load --input $(bazel cquery //enterprise/cmd/gitserver:image_tarball ${config} --output=files)
bazel build //cmd/gitserver:image_tarball ${config} && \
docker load --input $(bazel cquery //cmd/gitserver:image_tarball ${config} --output=files)
gitserver-0:
cmd: |
docker inspect gitserver-${GITSERVER_INDEX} >/dev/null 2>&1 && docker stop gitserver-${GITSERVER_INDEX}

85
enterprise/cmd/gitserver/BUILD.bazel
View File

@ -1,85 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("@container_structure_test//:defs.bzl", "container_structure_test")
load("//dev:oci_defs.bzl", "image_repository")
go_library(
name = "gitserver_lib",
srcs = ["main.go"],
importpath = "github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver",
visibility = ["//visibility:private"],
deps = [
"//enterprise/cmd/gitserver/shared",
"//enterprise/cmd/sourcegraph/enterprisecmd",
"//internal/sanitycheck",
],
)
go_binary(
name = "gitserver",
embed = [":gitserver_lib"],
visibility = ["//visibility:public"],
x_defs = {
"github.com/sourcegraph/sourcegraph/internal/version.version": "{STABLE_VERSION}",
"github.com/sourcegraph/sourcegraph/internal/version.timestamp": "{VERSION_TIMESTAMP}",
},
)
pkg_tar(
name = "tar_gitserver",
srcs = [":gitserver"],
)
pkg_tar(
name = "tar_p4_fusion_wrappers",
srcs = [
"p4-fusion-wrapper-detect-kill.sh",
"process-stats-watcher.sh",
],
remap_paths = {
"/p4-fusion-wrapper-detect-kill.sh": "/usr/local/bin/p4-fusion",
"/process-stats-watcher.sh": "/usr/local/bin/process-stats-watcher.sh",
},
visibility = ["//visibility:public"],
)
oci_image(
name = "image",
base = "@wolfi_gitserver_base",
entrypoint = [
"/sbin/tini",
"--",
"/gitserver",
],
tars = [
":tar_gitserver",
":tar_p4_fusion_wrappers",
],
user = "sourcegraph",
workdir = "/",
)
oci_tarball(
name = "image_tarball",
image = ":image",
repo_tags = ["gitserver:candidate"],
)
container_structure_test(
name = "image_test",
timeout = "short",
configs = ["image_test.yaml"],
driver = "docker",
image = ":image",
tags = [
"exclusive",
"requires-network",
],
)
oci_push(
name = "candidate_push",
image = ":image",
repository = image_repository("gitserver"),
)

68
enterprise/cmd/gitserver/image_test.yaml
View File

@ -1,68 +0,0 @@
schemaVersion: "2.0.0"
commandTests:
- name: "binary is runnable"
command: "/gitserver"
envVars:
- key: "SANITY_CHECK"
value: "true"
- name: "git is runnable"
command: "git"
args:
- version
- name: "git-lfs is runnable"
command: "git-lfs"
args:
- version
- name: "git p4 is runnable"
command: "git"
args:
- p4
expectedOutput: ["valid commands: submit"]
exitCode: 2
- name: "ssh is runnable"
command: "ssh"
exitCode: 255
- name: "python3 is runnable"
command: "python3"
args:
- --version
- name: "bash is runnable"
command: "bash"
args:
- --version
- name: "p4 is runnable"
command: "p4"
args:
- -h
- name: "coursier is runnable"
command: "coursier"
- name: "p4-fusion is runnable"
command: "p4-fusion-binary"
- name: "not running as root"
command: "/usr/bin/id"
args:
- -u
excludedOutput: ["^0"]
exitCode: 0
fileExistenceTests:
- name: '/data/repos'
path: '/data/repos'
shouldExist: true
uid: 100
gid: 101
# p4-fusion wrappers
- name: '/usr/local/bin/p4-fusion'
path: '/usr/local/bin/p4-fusion'
shouldExist: true
uid: 0
gid: 0
permissions: '-r-xr-xr-x'
- name: '/usr/local/bin/process-stats-watcher.sh'
path: '/usr/local/bin/process-stats-watcher.sh'
shouldExist: true
uid: 0
gid: 0
permissions: '-r-xr-xr-x'

13
enterprise/cmd/gitserver/main.go
View File

@ -1,13 +0,0 @@
// Command frontend is the enterprise frontend program.
package main
import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
"github.com/sourcegraph/sourcegraph/internal/sanitycheck"
)
func main() {
sanitycheck.Pass()
enterprisecmd.SingleServiceMainEnterprise(shared.Service)
}

75
enterprise/cmd/gitserver/p4-fusion-wrapper-detect-kill.sh
View File

@ -1,75 +0,0 @@
#!/usr/bin/env bash
# shellcheck disable=SC2064,SC2207
# create a file to hold the output of p4-fusion
# TODO: consider recording/storing/capturing the file for logs display in the UI if there's a problem
fusionout=$(mktemp || mktemp -t fusionout_XXXXXXXX)
# create a pipe to use for capturing output of p4-fusion
# so that it can be sent to stdout and also to a file for analyzing later
fusionpipe=$(mktemp || mktemp -t fusionpipe_XXXXXXXX)
rm -f "${fusionpipe}"
mknod "${fusionpipe}" p
tee <"${fusionpipe}" "${fusionout}" &
# create a file to hold the output of `wait`
waitout=$(mktemp || mktemp -t waitout_XXXXXXXX)
# create a file to hold the resource usage of the child process
stats=$(mktemp || mktemp -t resource_XXXXXXXX)
# make sure to clean up on exit
trap "rm -f \"${fusionout}\" \"${fusionpipe}\" \"${waitout}\" \"${stats}\"" EXIT
# launch p4-fusion in the background, sending all output to the pipe for capture and re-echoing
# depends on the p4-fusion binary executable being copied to p4-fusion-binary in the gitserver Dockerfile
p4-fusion-binary "${@}" >"${fusionpipe}" 2>&1 &
# capture the pid of the child process
fpid=$!
# start up a "sidecar" process to capture resource usage.
# it will terminate when the p4-fusion process terminates.
process-stats-watcher.sh "${fpid}" "p4-fusion-binary" >"${stats}" &
spid=$!
# Wait for the child process to finish
wait ${fpid} >"${waitout}" 2>&1
# capture the result of the wait, which is the result of the child process
# or the result of external action on the child process, like SIGKILL
waitcode=$?
# the sidecar process should have exited by now, but just in case, wait for it
wait "${spid}" >/dev/null 2>&1
[ ${waitcode} -eq 0 ] || {
# if the wait exit code indicates a problem,
# check to see if the child process was killed
killed=""
# if the process was killed with SIGKILL, the `wait` process will have generated a notification
grep -qs "Killed" "${waitout}" && killed=y
[ -z "${killed}" ] && {
# If the wait process did not generate an error message, check the process output.
# The process traps SIGINT and SIGTERM; uncaught signals will be displayed as "uncaught"
tail -5 "${fusionout}" | grep -Eqs "Signal Received:|uncaught target signal" && killed=y
}
[ -z "${killed}" ] || {
# include the signal if it's SIGINT, SIGTERM, or SIGKILL
# not gauranteed to work, but nice if we can include the info
signal="$(kill -l ${waitcode})"
[ -z "${signal}" ] || signal=" (SIG${signal})"
# get info if available from the sidecar process
rusage=""
[ -s "${stats}" ] && {
# expect the last (maybe only) line to be four fields:
# RSS VSZ ETIME TIME
x=($(tail -1 "${stats}"))
# NOTE: bash indexes from 0; zsh indexes from 1
[ ${#x[@]} -eq 4 ] && rusage=" At the time of its demise, it had been running for ${x[2]}, had used ${x[3]} CPU time, reserved ${x[1]} RAM and was using ${x[0]}."
}
echo "p4-fusion was killed by an external signal${signal}.${rusage}"
}
}
exit ${waitcode}

45
enterprise/cmd/gitserver/process-stats-watcher.sh
View File

@ -1,45 +0,0 @@
#!/usr/bin/env bash
# shellcheck disable=SC2064,SC2207,SC2009
humanize() {
local num=${1}
[[ ${num} =~ ^[0-9][0-9]*$ ]] && num=$(bc <<<"scale=2;${num}/1024/1024")m
printf -- '%s' "${num}"
return 0
}
# read resource usage statistics for a process
# several times a second until it terminates
# at which point, output the most recent stats on stdout
# the output format is "RSS VSZ ETIME TIME"
# input is the pid of the process
pid="${1}"
# and its name, which is used to avoid tracking
# another process in case the original process completed,
# and another started up and got assigned the same pid
cmd="${2}"
unset rss vsz etime time
while true; do
# Alpine has a rather limited `ps`
# it does not limit output to just one process, even when specifying a pid
# so we need to filter the output by pid
x="$(ps -o pid,stat,rss,vsz,etime,time,comm,args "${pid}" | grep "^ *${pid} " | grep "${cmd}" | tail -1)"
[ -z "${x}" ] && break
IFS=" " read -r -a a <<<"$x"
# drop out of here if the process has died or become a zombie - no coming back from the dead
[[ "${a[1]}" =~ ^[ZXx] ]] && break
# only collect stats for processes that are active (running, sleeping, disk sleep, which is waiting for I/O to complete)
# but don't stop until it is really is dead
[[ "${a[1]}" =~ ^[RSD] ]] && {
rss=${a[2]}
vsz=${a[3]}
etime=${a[4]}
time=${a[5]}
}
sleep 0.2
done
printf '%s %s %s %s' "$(humanize "${rss}")" "$(humanize "${vsz}")" "${etime}" "${time}"

23
enterprise/cmd/gitserver/shared/BUILD.bazel
View File

@ -1,23 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "shared",
srcs = [
"service.go",
"shared.go",
],
importpath = "github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver/shared",
visibility = ["//visibility:public"],
deps = [
"//cmd/gitserver/shared",
"//internal/authz",
"//internal/authz/subrepoperms",
"//internal/database",
"//internal/debugserver",
"//internal/encryption/keyring",
"//internal/env",
"//internal/observation",
"//internal/service",
"@com_github_sourcegraph_log//:log",
],
)

29
enterprise/cmd/gitserver/shared/service.go
View File

@ -1,29 +0,0 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/shared"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "gitserver" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
c := shared.LoadConfig()
endpoints := []debugserver.Endpoint{
shared.GRPCWebUIDebugEndpoint(c.ListenAddress),
}
return c, endpoints
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return shared.Main(ctx, observationCtx, ready, config.(*shared.Config), enterpriseInit)
}
var Service service.Service = svc{}

23
enterprise/cmd/gitserver/shared/shared.go
View File

@ -1,23 +0,0 @@
// Package shared is the enterprise gitserrver program's shared main entrypoint.
//
// It lets the invoker of the OSS gitserver shared entrypoint inject a few
// proprietary things into it via e.g. blank/underscore imports in this file
// which register side effects with the gitserver package.
package shared
import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/authz"
srp "github.com/sourcegraph/sourcegraph/internal/authz/subrepoperms"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/encryption/keyring"
)
func enterpriseInit(db database.DB, keyring keyring.Ring) {
logger := log.Scoped("enterprise", "gitserver enterprise edition")
var err error
authz.DefaultSubRepoPermsChecker, err = srp.NewSubRepoPermsClient(db.SubRepoPerms())
if err != nil {
logger.Fatal("Failed to create sub-repo client", log.Error(err))
}
}

4
enterprise/cmd/server/BUILD.bazel
View File

@ -115,7 +115,7 @@ DEPS = [
"//cmd/searcher",
"//enterprise/cmd/embeddings",
"//enterprise/cmd/frontend",
"//enterprise/cmd/gitserver",
"//cmd/gitserver",
"//enterprise/cmd/migrator",
"//cmd/repo-updater",
"//enterprise/cmd/symbols",
@ -180,7 +180,7 @@ oci_image(
":tar_postgres_optimize",
":tar_postgres_reindex",
":tar_prom-wrapper",
"//enterprise/cmd/gitserver:tar_p4_fusion_wrappers",
"//cmd/gitserver:tar_p4_fusion_wrappers",
"//monitoring:generate_grafana_config_tar",
"tar_image_test_scripts",
] + dependencies_tars(DEPS) + dependencies_tars(ZOEKT_DEPS),

2
enterprise/cmd/sourcegraph/BUILD.bazel
View File

@ -9,12 +9,12 @@ go_library(
"//cmd/blobstore/shared",
"//cmd/executor/singlebinary",
"//cmd/github-proxy/shared",
"//cmd/gitserver/shared",
"//cmd/precise-code-intel-worker/shared",
"//cmd/repo-updater/shared",
"//cmd/searcher/shared",
"//enterprise/cmd/embeddings/shared",
"//enterprise/cmd/frontend/shared",
"//enterprise/cmd/gitserver/shared",
"//enterprise/cmd/sourcegraph/enterprisecmd",
"//enterprise/cmd/symbols/shared",
"//enterprise/cmd/worker/shared",

2
enterprise/cmd/sourcegraph/main.go
View File

@ -12,12 +12,12 @@ import (
blobstore_shared "github.com/sourcegraph/sourcegraph/cmd/blobstore/shared"
executor_singlebinary "github.com/sourcegraph/sourcegraph/cmd/executor/singlebinary"
githubproxy_shared "github.com/sourcegraph/sourcegraph/cmd/github-proxy/shared"
gitserver_shared "github.com/sourcegraph/sourcegraph/cmd/gitserver/shared"
precise_code_intel_worker_shared "github.com/sourcegraph/sourcegraph/cmd/precise-code-intel-worker/shared"
repoupdater_shared "github.com/sourcegraph/sourcegraph/cmd/repo-updater/shared"
searcher_shared "github.com/sourcegraph/sourcegraph/cmd/searcher/shared"
embeddings_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/embeddings/shared"
frontend_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/shared"
gitserver_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver/shared"
symbols_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols/shared"
worker_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/shared"

5
sg.config.yaml
View File

@ -173,7 +173,7 @@ commands:
if [ -n "$DELVE" ]; then
export GCFLAGS='-N -l'
fi
go build -gcflags="$GCFLAGS" -o .bin/gitserver github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver
go build -gcflags="$GCFLAGS" -o .bin/gitserver github.com/sourcegraph/sourcegraph/cmd/gitserver
checkBinary: .bin/gitserver
env: &gitserverenv
HOSTNAME: 127.0.0.1:3178
@ -181,7 +181,6 @@ commands:
- lib
- internal
- cmd/gitserver
- enterprise/cmd/gitserver
# This is only here to stay backwards-compatible with people's custom
# `sg.config.overwrite.yaml` files
@ -945,7 +944,7 @@ bazelCommands:
CTAGS_PROCESSES: 2
USE_ROCKSKIP: "false"
gitserver-template: &gitserver_bazel_template
target: //enterprise/cmd/gitserver
target: //cmd/gitserver
env: &gitserverenv
HOSTNAME: 127.0.0.1:3178
# This is only here to stay backwards-compatible with people's custom