diff --git a/.github/workflows/universal-ctags.yml b/.github/workflows/universal-ctags.yml new file mode 100644 index 00000000000..d68d6c0ba15 --- /dev/null +++ b/.github/workflows/universal-ctags.yml @@ -0,0 +1,80 @@ +name: universal-ctags + +on: + push: + paths: + - 'dev/nix/ctags.nix' + workflow_dispatch: +permissions: + contents: 'read' + id-token: 'write' + +jobs: + x86_64-darwin: + name: Build ctags + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v4 + - id: auth + name: '🔓 Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{ secrets.CTAGS_GCP_SERVICE_ACCOUNT }} + - id: nix-build + name: Run `nix build` + run: | + nix build .#ctags + sudo codesign --force -s - ./result/bin/universal-ctags-* + - id: 'upload-file' + uses: 'google-github-actions/upload-cloud-storage@v1' + with: + path: './result/bin/' + destination: 'universal_ctags/x86_64-darwin/' + glob: 'universal-ctags-*' + aarch64-darwin: + name: Build ctags + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v4 + - id: auth + name: '🔓 Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{ secrets.CTAGS_GCP_SERVICE_ACCOUNT }} + - id: nix-build + name: Run `nix build` + run: | + nix build .#ctags-aarch64-darwin + sudo codesign --force -s - ./result/bin/universal-ctags-* + - id: 'upload-file' + uses: 'google-github-actions/upload-cloud-storage@v1' + with: + path: './result/bin/' + destination: 'universal_ctags/aarch64-darwin' + glob: 'universal-ctags-*' + x86_64-linux: + name: Build ctags + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v4 + - id: auth + name: '🔓 Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{ secrets.CTAGS_GCP_SERVICE_ACCOUNT }} + - id: nix-build + name: Run `nix build` + run: | + nix build .#ctags + - id: 'upload-file' + uses: 'google-github-actions/upload-cloud-storage@v1' + with: + path: './result/bin/' + destination: 'universal_ctags/x86_64-linux' + glob: 'universal-ctags-*' diff --git a/dev/ctags-install.sh b/dev/ctags-install.sh deleted file mode 100755 index ff295941c66..00000000000 --- a/dev/ctags-install.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -set -euf -o pipefail -pushd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null -mkdir -p .bin - -# Commit hash of github.com/universal-ctags/ctags. -# Last bumped 2022-04-04. -# When bumping please remember to also update Zoekt: https://github.com/sourcegraph/zoekt/blob/d3a8fbd8385f0201dd54ab24114ebd588dfcf0d8/install-ctags-alpine.sh -CTAGS_VERSION=f95bb3497f53748c2b6afc7f298cff218103ab90 -NAME="ctags-${CTAGS_VERSION}" -TARGET="$PWD/.bin/${NAME}" - -if [ $# -ne 0 ]; then - if [ "$1" == "which" ]; then - echo "$TARGET" - exit 0 - fi -fi - -tmpdir=$(mktemp -d -t sg.ctags-install.XXXXXX) - -function print_build_tips() { - echo "---------------------------------------------" - echo "Please make sure that libjansson is installed" - echo " MacOs: brew install jansson" - echo " Ubuntu: apt-get install libjansson-dev" - echo "---------------------------------------------" -} - -function ctrl_c() { - rm -f "$tmpdir" &>/dev/null - printf "[-] Installation cancelled.\n" - exit 1 -} - -trap ctrl_c INT -trap 'rm -Rf \"$tmpdir\" &>/dev/null' EXIT - -function build_ctags { - case "$OSTYPE" in - darwin*) NUMCPUS=$(sysctl -n hw.ncpu);; - linux*) NUMCPUS=$(grep -c '^processor' /proc/cpuinfo) ;; - bsd*) NUMCPUS=$(grep -c '^processor' /proc/cpuinfo) ;; - *) NUMCPUS="4" ;; - esac - - curl --retry 5 "https://codeload.github.com/universal-ctags/ctags/tar.gz/$CTAGS_VERSION" | tar xz -C "$tmpdir" - cd "${tmpdir}/ctags-${CTAGS_VERSION}" - set +e - ./autogen.sh - exit_code="$?" - if [ "$exit_code" != "0" ]; then - print_build_tips - exit "$exit_code" - fi - ./configure --program-prefix=universal- --enable-json - exit_code="$?" - if [ "$exit_code" != "0" ]; then - print_build_tips - exit "$exit_code" - fi - make -j"$NUMCPUS" --load-average="$NUMCPUS" - exit_code="$?" - if [ "$exit_code" != "0" ]; then - print_build_tips - exit "$exit_code" - fi - set -e - cp ./ctags "$TARGET" -} - -if [ ! -f "${TARGET}" ]; then - echo "Installing universal-ctags $CTAGS_VERSION" - build_ctags -else - echo "universal-ctags $CTAGS_VERSION is already available at $TARGET" -fi - -popd >/dev/null diff --git a/dev/nix/comby.nix b/dev/nix/comby.nix index 36fbc378c50..710443393a6 100644 --- a/dev/nix/comby.nix +++ b/dev/nix/comby.nix @@ -20,7 +20,7 @@ let }); in if hostPlatform.isMacOS then - unNixifyDylibs pkgs (combyBuilder pkgs) + unNixifyDylibs { inherit pkgs; } (combyBuilder pkgs) else # ocaml in pkgsStatic is problematic, so we use it from pkgsMusl instead and just # supply pkgsStatic system libraries such as openssl etc diff --git a/dev/nix/ctags.nix b/dev/nix/ctags.nix index 98714037514..f696af71aef 100644 --- a/dev/nix/ctags.nix +++ b/dev/nix/ctags.nix @@ -18,7 +18,7 @@ let stdenv = pkgsStatic.stdenv; in # yoinked from github.com/nixos/nixpkgs -unNixifyDylibs pkgs (stdenv.mkDerivation rec { +unNixifyDylibs { inherit pkgs; } (stdenv.mkDerivation rec { pname = "universal-ctags"; version = "5.9.20220403.0"; @@ -63,7 +63,7 @@ unNixifyDylibs pkgs (stdenv.mkDerivation rec { ''; postFixup = '' - ln -s $out/bin/ctags $out/bin/universal-ctags + ln -s $out/bin/ctags $out/bin/universal-ctags-$version ''; doCheck = true; diff --git a/dev/nix/p4-fusion.nix b/dev/nix/p4-fusion.nix index e5e5263150f..af716b33619 100644 --- a/dev/nix/p4-fusion.nix +++ b/dev/nix/p4-fusion.nix @@ -9,7 +9,7 @@ , patchelf , pkg-config , darwin -, targetPlatform +, hostPlatform }: let inherit (import ./util.nix { inherit lib; }) mkStatic unNixifyDylibs; @@ -31,7 +31,7 @@ let # pkgsStatic.zlib.static doesn't exist on linux, but does on macos zlib-static = (pkgsStatic.zlib.static or pkgsStatic.zlib); in -unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec { +unNixifyDylibs { inherit pkgs; } (pkgsStatic.gccStdenv.mkDerivation rec { name = "p4-fusion"; version = "v1.12"; @@ -44,8 +44,8 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec { hash = "sha256-rUXuBoXuOUanWxutd7dNgjn2vLFvHQ0IgCIn9vG5dgs="; }) ( - if targetPlatform.isMacOS then - if targetPlatform.isAarch64 then + if hostPlatform.isMacOS then + if hostPlatform.isAarch64 then fetchzip { name = "helix-core-api"; @@ -58,14 +58,14 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec { url = "https://cdist2.perforce.com/perforce/r22.2/bin.macosx12x86_64/p4api-openssl3.tgz"; hash = "sha256-/Ia9R+H95Yx4Sx7+Grke0d3QskuZ2YtH4LZOS7vRMZc="; } - else if targetPlatform.isLinux then + else if hostPlatform.isLinux then fetchzip { name = "helix-core-api"; url = "https://cdist2.perforce.com/perforce/r22.2/bin.linux26x86_64/p4api-glibc2.3-openssl3.tgz"; hash = "sha256-tqWhdQQdOVAiGa6HiRajw4emoYRRRgZf6pZVEIf1qqU="; } - else throw "unsupported platform ${stdenv.targetPlatform.parsed.kernel.name}" + else throw "unsupported platform ${stdenv.hostPlatform.parsed.kernel.name}" ) ]; @@ -82,7 +82,7 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec { http-parser-static pcre-static openssl-static - ] ++ lib.optional targetPlatform.isMacOS [ + ] ++ lib.optional hostPlatform.isMacOS [ # iconv is bundled with glibc and apparently only needed for osx # https://sourcegraph.com/github.com/salesforce/p4-fusion@3ee482466464c18e6a635ff4f09cd75a2e1bfe0f/-/blob/vendor/libgit2/README.md?L178:3 libiconv-static @@ -91,7 +91,7 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec { ]; # copy helix-core-api stuff into the expected directories, and statically link libstdc++ - preBuild = let dir = if targetPlatform.isMacOS then "mac" else "linux"; in + preBuild = let dir = if hostPlatform.isMacOS then "mac" else "linux"; in '' mkdir -p $NIX_BUILD_TOP/$sourceRoot/vendor/helix-core-api/${dir} cp -R $NIX_BUILD_TOP/helix-core-api/* $NIX_BUILD_TOP/$sourceRoot/vendor/helix-core-api/${dir} diff --git a/dev/nix/util.nix b/dev/nix/util.nix index c50e1afb295..9bcff57ddae 100644 --- a/dev/nix/util.nix +++ b/dev/nix/util.nix @@ -1,36 +1,65 @@ { lib }: { # utility function to add some best-effort flags for emitting static objects instead of dynamic - mkStatic = pkg: + mkStatic = drv: + assert lib.assertMsg (lib.isDerivation drv) "mkStatic expects a derivation, got ${builtins.typeOf drv}"; + assert lib.assertMsg (drv ? "overrideAttrs") "mkStatic expects an overridable derivation"; + let - auto = builtins.intersectAttrs pkg.override.__functionArgs { withStatic = true; static = true; enableStatic = true; enableShared = false; }; - overridden = pkg.overrideAttrs (oldAttrs: { + auto = builtins.intersectAttrs drv.override.__functionArgs { withStatic = true; static = true; enableStatic = true; enableShared = false; }; + overridden = drv.overrideAttrs (oldAttrs: { dontDisableStatic = true; } // lib.optionalAttrs (!(oldAttrs.dontAddStaticConfigureFlags or false)) { configureFlags = (oldAttrs.configureFlags or [ ]) ++ [ "--disable-shared" "--enable-static" "--enable-shared=false" ]; }); in - if pkg.pname == "openssl" then pkg.override { static = true; } else overridden.override auto; + if drv.pname == "openssl" then drv.override { static = true; } else overridden.override auto; # doesn't actually change anything in practice, just makes otool -L not display nix store paths for libiconv and libxml. # they exist in macos dydl cache anyways, so where they point to is irrelevant. worst case, this will let you catch earlier # when a library that should be statically linked or that isnt in dydl cache is dynamically linked. - unNixifyDylibs = pkgs: drv: - drv.overrideAttrs (oldAttrs: { - postFixup = with pkgs; (oldAttrs.postFixup or "") + lib.optionalString pkgs.hostPlatform.isMacOS '' - for bin in $(${findutils}/bin/find $out/bin -type f); do - for lib in $(otool -L $bin | ${coreutils}/bin/tail -n +2 | ${coreutils}/bin/cut -d' ' -f1 | ${gnugrep}/bin/grep nix); do + unNixifyDylibs = { pkgs }: drv: + assert lib.assertMsg (lib.isDerivation drv) "unNixifyDylibs expects a derivation, got ${builtins.typeOf drv}"; + assert lib.assertMsg (drv ? "overrideAttrs") "unNixifyDylibs expects an overridable derivation"; + + drv.overrideAttrs (oldAttrs: lib.optionalAttrs pkgs.hostPlatform.isMacOS { + nativeBuildInputs = (oldAttrs.nativeBuildInputs or [ ]) ++ + map (drv: drv.__spliced.buildHost or drv) + (with (pkgs.__splicedPackages or pkgs); [ + findutils + darwin.cctools + coreutils + gnugrep + ]); + + postFixup = (oldAttrs.postFixup or "") + '' + for bin in $(find $out/bin -type f); do + for lib in $(otool -L $bin | tail -n +2 | cut -d' ' -f1 | grep nix); do + echo "patching dylib from "$lib" to "@rpath/$(basename $lib)"" install_name_tool -change "$lib" "@rpath/$(basename $lib)" $bin done done ''; }); - # removes packages from a list of packages by name. - # Copied from https://sourcegraph.com/github.com/NixOS/nixpkgs@4d924a6b3376c5e3cae3ba8c971007bf736084c5/-/blob/nixos/lib/utils.nix?L219 - removePackagesByName = packages: packagesToRemove: - let - namesToRemove = map lib.getName packagesToRemove; - in - lib.filter (x: !(builtins.elem (lib.getName x) namesToRemove)) packages; + # returns a set of unsuffixed derivations for a native target and suffixed derivations for an optional cross-compile target + # returned by applying `f` to the passed native & cross-compile package sets. + xcompilify = { pkgs, pkgsX }: f: lib.foldl # merge all outputs into a single set + (acc: pkgSet: acc // + # rename with -${xtarget} if an xtarget package + (lib.mapAttrs' + (name: drv: + assert lib.assertMsg (lib.isDerivation drv) "expected derivation, got ${builtins.typeOf drv}"; { + name = (name + lib.optionalString + # cant use drv.stdenv.buildPlatform.system here, as + # aarch64-darwin.pkgsx86_64Darwin.stdenv.buildPlatform.system == aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.system + (pkgs.system != drv.stdenv.hostPlatform.system) "-${drv.stdenv.hostPlatform.system}" + ); + value = drv; + }) + pkgSet) + ) + { } + # call f with native pkgs (and, if non-null, cross-compile pkgsx) + (builtins.map (pkgs: f pkgs) ([ pkgs ] ++ lib.optional (pkgsX != null) pkgsX)); } diff --git a/dev/tool_deps.bzl b/dev/tool_deps.bzl index a8c304e2cf9..edd3abcd251 100644 --- a/dev/tool_deps.bzl +++ b/dev/tool_deps.bzl @@ -1,7 +1,8 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file", "http_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") DOCSITE_VERSION = "1.9.3" SRC_CLI_VERSION = "5.1.0" +CTAGS_VERSION = "5.9.20220403.0" SRC_CLI_BUILDFILE = """ filegroup( @@ -56,3 +57,24 @@ def tool_deps(): url = "https://github.com/sourcegraph/src-cli/releases/download/{0}/src-cli_{0}_darwin_arm64.tar.gz".format(SRC_CLI_VERSION), ) + # universal-ctags # + http_file( + name = "universal-ctags-darwin-x86_64", + sha256 = "b69501d497b62021e8438e840e0bea62fdbe91d60cf8375c388f2736cd58a1bf", + url = "https://storage.googleapis.com/universal_ctags/x86_64-darwin/bin/universal-ctags-{0}".format(CTAGS_VERSION), + executable = True, + ) + + http_file( + name = "universal-ctags-darwin-arm64", + sha256 = "51b3b7ea296455e00fc5a7aafea49bb89551e81770b3728c97a04a5614fde8c5", + url = "https://storage.googleapis.com/universal_ctags/aarch64-darwin/bin/universal-ctags-{0}".format(CTAGS_VERSION), + executable = True, + ) + + http_file( + name = "universal-ctags-linux-amd64", + sha256 = "1d349d15736a30c9cc18c1fd9efbfc6081fb59125d799b84cef6b34c735fa28a", + url = "https://storage.googleapis.com/universal_ctags/x86_64-linux/bin/universal-ctags-{0}".format(CTAGS_VERSION), + executable = True, + ) diff --git a/dev/tools/BUILD.bazel b/dev/tools/BUILD.bazel index dc6a1569469..94f3aeea54e 100644 --- a/dev/tools/BUILD.bazel +++ b/dev/tools/BUILD.bazel @@ -17,3 +17,13 @@ sh_binary( }), visibility = ["//visibility:public"], ) + +sh_binary( + name = "universal-ctags", + srcs = select({ + "@bazel_tools//src/conditions:darwin_x86_64": ["@universal-ctags-darwin-amd64//file:downloaded"], + "@bazel_tools//src/conditions:darwin_arm64": ["@universal-ctags-darwin-arm64//file:downloaded"], + "@bazel_tools//src/conditions:linux_x86_64": ["@universal-ctags-linux-amd64//file:downloaded"], + }), + visibility = ["//visibility:public"], +) diff --git a/dev/universal-ctags-dev b/dev/universal-ctags-dev index 0d3b10bf05b..0e260a2e336 100755 --- a/dev/universal-ctags-dev +++ b/dev/universal-ctags-dev @@ -1,22 +1,8 @@ #!/usr/bin/env bash -# This script is a wrapper around `universal-ctags`. -# -# It checks if universal-ctags has been installed through ./dev/ctags-install.sh or falls back to a dockerized version. +# This script is a wrapper around `universal-ctags` to download and run universal-ctags using bazel # # To use your own `universal-ctags` binary instead of this wrapper in your local dev server, use # `CTAGS_COMMAND=path/to/ctags sg start`. -root="$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null -TARGET=$("$root/dev/ctags-install.sh" which) - -if [ ! -f "${TARGET}" ]; then - exec docker run --rm -i \ - -a stdin -a stdout -a stderr \ - --user guest \ - --name=universal-ctags-$$ \ - --entrypoint /usr/local/bin/universal-ctags \ - ctags "$@" -else - ${TARGET} "$@" -fi +bazel run //dev/tools:universal-ctags -- "$@" diff --git a/flake.lock b/flake.lock index c4bdf0d7c1a..d07ef7470ad 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1681202837, - "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", "owner": "numtide", "repo": "flake-utils", - "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", "type": "github" }, "original": { @@ -28,9 +28,10 @@ "type": "github" }, "original": { - "id": "nixpkgs", - "ref": "nixos-unstable", - "type": "indirect" + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e78d25df6f1036b3fa76750ed4603dd9d5fe90fc", + "type": "github" } }, "root": { diff --git a/flake.nix b/flake.nix index 8d47dbbef19..4d080185121 100644 --- a/flake.nix +++ b/flake.nix @@ -2,30 +2,35 @@ description = "The Sourcegraph developer environment & packages Nix Flake"; inputs = { - nixpkgs.url = "nixpkgs/nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs/e78d25df6f1036b3fa76750ed4603dd9d5fe90fc"; flake-utils.url = "github:numtide/flake-utils"; }; outputs = { self, nixpkgs, flake-utils }: + let + xcompileTargets = with nixpkgs.lib.systems.examples; { + "aarch64-darwin" = nixpkgs.legacyPackages.aarch64-darwin.pkgsx86_64Darwin; + "x86_64-darwin" = import nixpkgs { system = "x86_64-darwin"; crossSystem = aarch64-darwin; }; + }; + inherit (import ./dev/nix/util.nix { inherit (nixpkgs) lib; }) xcompilify; + in flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; pkgs' = import nixpkgs { inherit system; overlays = builtins.attrValues self.overlays; }; + pkgsX = xcompileTargets.${system} or null; in { - # We set legacyPackages to our custom static binaries so command - # like "nix build .#p4-fusion" work. legacyPackages = pkgs'; - packages = { - ctags = pkgs.callPackage ./dev/nix/ctags.nix { }; - comby = pkgs.callPackage ./dev/nix/comby.nix { }; + packages = xcompilify { inherit pkgs pkgsX; } + (pkgs: { + ctags = pkgs.callPackage ./dev/nix/ctags.nix { }; + comby = pkgs.callPackage ./dev/nix/comby.nix { }; + p4-fusion = pkgs.callPackage ./dev/nix/p4-fusion.nix { }; + }) // { nodejs-16_x = pkgs.callPackage ./dev/nix/nodejs.nix { }; - } - # so we don't get `packages.aarch64-linux.p4-fusion` in nix `flake show` output - // pkgs.lib.optionalAttrs (pkgs.targetPlatform.system != "aarch64-linux") { - p4-fusion = pkgs.callPackage ./dev/nix/p4-fusion.nix { }; }; # We use pkgs (not pkgs') intentionally to avoid doing extra work of diff --git a/sg.config.yaml b/sg.config.yaml index de1e297eb90..e872267a61b 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -239,7 +239,6 @@ commands: export GCFLAGS='-N -l' fi - ./cmd/symbols/build-ctags.sh && go build -gcflags="$GCFLAGS" -o .bin/symbols github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols checkBinary: .bin/symbols env: @@ -927,7 +926,6 @@ bazelCommands: env: ENTERPRISE: 1 symbols: - # TODO build ctags thing target: //enterprise/cmd/symbols checkBinary: .bin/symbols env: