sg: use bazel run universtal-ctags (#55944)

Use nix to compile and upload universal-ctags to a gcloud bucket. We then pull this down using a bazel http_file and can run it with bazel also. Removed all of the old bash logic that was hard to follow and maintain also. --------- Co-authored-by: Noah Santschi-Cooney <noah@santschi-cooney.ch> Co-authored-by: William Bezuidenhout <william.bezuidenhout@sourcegraph.com>
2026-02-06 17:31:43 +00:00 · 2023-08-18 11:02:46 -05:00 · 2023-08-18 11:02:46 -05:00 · f882023227
commit f882023227
parent 03e180aea2
12 changed files with 193 additions and 142 deletions
--- a/.github/workflows/universal-ctags.yml
+++ b/.github/workflows/universal-ctags.yml
@ -0,0 +1,80 @@
+name: universal-ctags
+
+on:
+  push:
+    paths:
+    - 'dev/nix/ctags.nix'
+  workflow_dispatch:
+permissions:
+  contents: 'read'
+  id-token: 'write'
+
+jobs:
+  x86_64-darwin:
+    name: Build ctags
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+      - id: auth
+        name: '🔓 Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.CTAGS_GCP_SERVICE_ACCOUNT }}
+      - id: nix-build
+        name: Run `nix build`
+        run: |
+          nix build .#ctags
+          sudo codesign --force -s - ./result/bin/universal-ctags-*
+      - id: 'upload-file'
+        uses: 'google-github-actions/upload-cloud-storage@v1'
+        with:
+          path: './result/bin/'
+          destination: 'universal_ctags/x86_64-darwin/'
+          glob: 'universal-ctags-*'
+  aarch64-darwin:
+    name: Build ctags
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+      - id: auth
+        name: '🔓 Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.CTAGS_GCP_SERVICE_ACCOUNT }}
+      - id: nix-build
+        name: Run `nix build`
+        run: |
+          nix build .#ctags-aarch64-darwin
+          sudo codesign --force -s - ./result/bin/universal-ctags-*
+      - id: 'upload-file'
+        uses: 'google-github-actions/upload-cloud-storage@v1'
+        with:
+          path: './result/bin/'
+          destination: 'universal_ctags/aarch64-darwin'
+          glob: 'universal-ctags-*'
+  x86_64-linux:
+    name: Build ctags
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+      - id: auth
+        name: '🔓 Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.CTAGS_GCP_SERVICE_ACCOUNT }}
+      - id: nix-build
+        name: Run `nix build`
+        run: |
+          nix build .#ctags
+      - id: 'upload-file'
+        uses: 'google-github-actions/upload-cloud-storage@v1'
+        with:
+          path: './result/bin/'
+          destination: 'universal_ctags/x86_64-linux'
+          glob: 'universal-ctags-*'
--- a/dev/ctags-install.sh
+++ b/dev/ctags-install.sh
@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-
-set -euf -o pipefail
-pushd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null
-mkdir -p .bin
-
-# Commit hash of github.com/universal-ctags/ctags.
-# Last bumped 2022-04-04.
-# When bumping please remember to also update Zoekt: https://github.com/sourcegraph/zoekt/blob/d3a8fbd8385f0201dd54ab24114ebd588dfcf0d8/install-ctags-alpine.sh
-CTAGS_VERSION=f95bb3497f53748c2b6afc7f298cff218103ab90
-NAME="ctags-${CTAGS_VERSION}"
-TARGET="$PWD/.bin/${NAME}"
-
-if [ $# -ne 0 ]; then
-  if [ "$1" == "which" ]; then
-    echo "$TARGET"
-    exit 0
-  fi
-fi
-
-tmpdir=$(mktemp -d -t sg.ctags-install.XXXXXX)
-
-function print_build_tips() {
-  echo "---------------------------------------------"
-  echo "Please make sure that libjansson is installed"
-  echo "  MacOs: brew install jansson"
-  echo "  Ubuntu: apt-get install libjansson-dev"
-  echo "---------------------------------------------"
-}
-
-function ctrl_c() {
-  rm -f "$tmpdir" &>/dev/null
-  printf "[-] Installation cancelled.\n"
-  exit 1
-}
-
-trap ctrl_c INT
-trap 'rm -Rf \"$tmpdir\" &>/dev/null' EXIT
-
-function build_ctags {
-  case "$OSTYPE" in
-    darwin*)  NUMCPUS=$(sysctl -n hw.ncpu);;
-    linux*)   NUMCPUS=$(grep -c '^processor' /proc/cpuinfo) ;;
-    bsd*)     NUMCPUS=$(grep -c '^processor' /proc/cpuinfo) ;;
-    *)        NUMCPUS="4" ;;
-  esac
-
-  curl --retry 5 "https://codeload.github.com/universal-ctags/ctags/tar.gz/$CTAGS_VERSION" | tar xz -C "$tmpdir"
-  cd "${tmpdir}/ctags-${CTAGS_VERSION}"
-  set +e
-  ./autogen.sh
-  exit_code="$?"
-  if [ "$exit_code" != "0" ]; then
-    print_build_tips
-    exit "$exit_code"
-  fi
-  ./configure --program-prefix=universal- --enable-json
-  exit_code="$?"
-  if [ "$exit_code" != "0" ]; then
-    print_build_tips
-    exit "$exit_code"
-  fi
-  make -j"$NUMCPUS" --load-average="$NUMCPUS"
-  exit_code="$?"
-  if [ "$exit_code" != "0" ]; then
-    print_build_tips
-    exit "$exit_code"
-  fi
-  set -e
-  cp ./ctags "$TARGET"
-}
-
-if [ ! -f "${TARGET}" ]; then
-  echo "Installing universal-ctags $CTAGS_VERSION"
-  build_ctags
-else
-  echo "universal-ctags $CTAGS_VERSION is already available at $TARGET"
-fi
-
-popd >/dev/null
--- a/dev/nix/comby.nix
+++ b/dev/nix/comby.nix
@ -20,7 +20,7 @@ let
    });
 in
 if hostPlatform.isMacOS then
-  unNixifyDylibs pkgs (combyBuilder pkgs)
+  unNixifyDylibs { inherit pkgs; } (combyBuilder pkgs)
 else
 # ocaml in pkgsStatic is problematic, so we use it from pkgsMusl instead and just
 # supply pkgsStatic system libraries such as openssl etc
--- a/dev/nix/ctags.nix
+++ b/dev/nix/ctags.nix
@ -18,7 +18,7 @@ let
  stdenv = pkgsStatic.stdenv;
 in
 # yoinked from github.com/nixos/nixpkgs
-unNixifyDylibs pkgs (stdenv.mkDerivation rec {
+unNixifyDylibs { inherit pkgs; } (stdenv.mkDerivation rec {
  pname = "universal-ctags";
  version = "5.9.20220403.0";

@ -63,7 +63,7 @@ unNixifyDylibs pkgs (stdenv.mkDerivation rec {
  '';

  postFixup = ''
-    ln -s $out/bin/ctags $out/bin/universal-ctags
+    ln -s $out/bin/ctags $out/bin/universal-ctags-$version
  '';

  doCheck = true;
--- a/dev/nix/p4-fusion.nix
+++ b/dev/nix/p4-fusion.nix
@ -9,7 +9,7 @@
 , patchelf
 , pkg-config
 , darwin
-, targetPlatform
+, hostPlatform
 }:
 let
  inherit (import ./util.nix { inherit lib; }) mkStatic unNixifyDylibs;
@ -31,7 +31,7 @@ let
  # pkgsStatic.zlib.static doesn't exist on linux, but does on macos
  zlib-static = (pkgsStatic.zlib.static or pkgsStatic.zlib);
 in
-unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec {
+unNixifyDylibs { inherit pkgs; } (pkgsStatic.gccStdenv.mkDerivation rec {
  name = "p4-fusion";
  version = "v1.12";

@ -44,8 +44,8 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec {
      hash = "sha256-rUXuBoXuOUanWxutd7dNgjn2vLFvHQ0IgCIn9vG5dgs=";
    })
    (
-      if targetPlatform.isMacOS then
-        if targetPlatform.isAarch64 then
+      if hostPlatform.isMacOS then
+        if hostPlatform.isAarch64 then
          fetchzip
            {
              name = "helix-core-api";
@ -58,14 +58,14 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec {
            url = "https://cdist2.perforce.com/perforce/r22.2/bin.macosx12x86_64/p4api-openssl3.tgz";
            hash = "sha256-/Ia9R+H95Yx4Sx7+Grke0d3QskuZ2YtH4LZOS7vRMZc=";
          }
-      else if targetPlatform.isLinux then
+      else if hostPlatform.isLinux then
        fetchzip
          {
            name = "helix-core-api";
            url = "https://cdist2.perforce.com/perforce/r22.2/bin.linux26x86_64/p4api-glibc2.3-openssl3.tgz";
            hash = "sha256-tqWhdQQdOVAiGa6HiRajw4emoYRRRgZf6pZVEIf1qqU=";
          }
-      else throw "unsupported platform ${stdenv.targetPlatform.parsed.kernel.name}"
+      else throw "unsupported platform ${stdenv.hostPlatform.parsed.kernel.name}"
    )
  ];

@ -82,7 +82,7 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec {
    http-parser-static
    pcre-static
    openssl-static
-  ] ++ lib.optional targetPlatform.isMacOS [
+  ] ++ lib.optional hostPlatform.isMacOS [
    # iconv is bundled with glibc and apparently only needed for osx
    # https://sourcegraph.com/github.com/salesforce/p4-fusion@3ee482466464c18e6a635ff4f09cd75a2e1bfe0f/-/blob/vendor/libgit2/README.md?L178:3
    libiconv-static
@ -91,7 +91,7 @@ unNixifyDylibs pkgs (pkgsStatic.gccStdenv.mkDerivation rec {
  ];

  # copy helix-core-api stuff into the expected directories, and statically link libstdc++
-  preBuild = let dir = if targetPlatform.isMacOS then "mac" else "linux"; in
+  preBuild = let dir = if hostPlatform.isMacOS then "mac" else "linux"; in
    ''
      mkdir -p $NIX_BUILD_TOP/$sourceRoot/vendor/helix-core-api/${dir}
      cp -R $NIX_BUILD_TOP/helix-core-api/* $NIX_BUILD_TOP/$sourceRoot/vendor/helix-core-api/${dir}
--- a/dev/nix/util.nix
+++ b/dev/nix/util.nix
@ -1,36 +1,65 @@
 { lib }:
 {
  # utility function to add some best-effort flags for emitting static objects instead of dynamic
-  mkStatic = pkg:
+  mkStatic = drv:
+    assert lib.assertMsg (lib.isDerivation drv) "mkStatic expects a derivation, got ${builtins.typeOf drv}";
+    assert lib.assertMsg (drv ? "overrideAttrs") "mkStatic expects an overridable derivation";
+
    let
-      auto = builtins.intersectAttrs pkg.override.__functionArgs { withStatic = true; static = true; enableStatic = true; enableShared = false; };
-      overridden = pkg.overrideAttrs (oldAttrs: {
+      auto = builtins.intersectAttrs drv.override.__functionArgs { withStatic = true; static = true; enableStatic = true; enableShared = false; };
+      overridden = drv.overrideAttrs (oldAttrs: {
        dontDisableStatic = true;
      } // lib.optionalAttrs (!(oldAttrs.dontAddStaticConfigureFlags or false)) {
        configureFlags = (oldAttrs.configureFlags or [ ]) ++ [ "--disable-shared" "--enable-static" "--enable-shared=false" ];
      });
    in
-    if pkg.pname == "openssl" then pkg.override { static = true; } else overridden.override auto;
+    if drv.pname == "openssl" then drv.override { static = true; } else overridden.override auto;

  # doesn't actually change anything in practice, just makes otool -L not display nix store paths for libiconv and libxml.
  # they exist in macos dydl cache anyways, so where they point to is irrelevant. worst case, this will let you catch earlier
  # when a library that should be statically linked or that isnt in dydl cache is dynamically linked.
-  unNixifyDylibs = pkgs: drv:
-    drv.overrideAttrs (oldAttrs: {
-      postFixup = with pkgs; (oldAttrs.postFixup or "") + lib.optionalString pkgs.hostPlatform.isMacOS ''
-        for bin in $(${findutils}/bin/find $out/bin -type f); do
-          for lib in $(otool -L $bin | ${coreutils}/bin/tail -n +2 | ${coreutils}/bin/cut -d' ' -f1 | ${gnugrep}/bin/grep nix); do
+  unNixifyDylibs = { pkgs }: drv:
+    assert lib.assertMsg (lib.isDerivation drv) "unNixifyDylibs expects a derivation, got ${builtins.typeOf drv}";
+    assert lib.assertMsg (drv ? "overrideAttrs") "unNixifyDylibs expects an overridable derivation";
+
+    drv.overrideAttrs (oldAttrs: lib.optionalAttrs pkgs.hostPlatform.isMacOS {
+      nativeBuildInputs = (oldAttrs.nativeBuildInputs or [ ]) ++
+        map (drv: drv.__spliced.buildHost or drv)
+          (with (pkgs.__splicedPackages or pkgs); [
+            findutils
+            darwin.cctools
+            coreutils
+            gnugrep
+          ]);
+
+      postFixup = (oldAttrs.postFixup or "") + ''
+        for bin in $(find $out/bin -type f); do
+          for lib in $(otool -L $bin | tail -n +2 | cut -d' ' -f1 | grep nix); do
+            echo "patching dylib from "$lib" to "@rpath/$(basename $lib)""
            install_name_tool -change "$lib" "@rpath/$(basename $lib)" $bin
          done
        done
      '';
    });

-  # removes packages from a list of packages by name.
-  # Copied from https://sourcegraph.com/github.com/NixOS/nixpkgs@4d924a6b3376c5e3cae3ba8c971007bf736084c5/-/blob/nixos/lib/utils.nix?L219
-  removePackagesByName = packages: packagesToRemove:
-    let
-      namesToRemove = map lib.getName packagesToRemove;
-    in
-    lib.filter (x: !(builtins.elem (lib.getName x) namesToRemove)) packages;
+  # returns a set of unsuffixed derivations for a native target and suffixed derivations for an optional cross-compile target
+  # returned by applying `f` to the passed native & cross-compile package sets.
+  xcompilify = { pkgs, pkgsX }: f: lib.foldl # merge all outputs into a single set
+    (acc: pkgSet: acc //
+      # rename with -${xtarget} if an xtarget package
+      (lib.mapAttrs'
+        (name: drv:
+          assert lib.assertMsg (lib.isDerivation drv) "expected derivation, got ${builtins.typeOf drv}"; {
+            name = (name + lib.optionalString
+              # cant use drv.stdenv.buildPlatform.system here, as
+              # aarch64-darwin.pkgsx86_64Darwin.stdenv.buildPlatform.system == aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.system
+              (pkgs.system != drv.stdenv.hostPlatform.system) "-${drv.stdenv.hostPlatform.system}"
+            );
+            value = drv;
+          })
+        pkgSet)
+    )
+    { }
+    # call f with native pkgs (and, if non-null, cross-compile pkgsx)
+    (builtins.map (pkgs: f pkgs) ([ pkgs ] ++ lib.optional (pkgsX != null) pkgsX));
 }
--- a/dev/tool_deps.bzl
+++ b/dev/tool_deps.bzl
@ -1,7 +1,8 @@
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")

 DOCSITE_VERSION = "1.9.3"
 SRC_CLI_VERSION = "5.1.0"
+CTAGS_VERSION = "5.9.20220403.0"

 SRC_CLI_BUILDFILE = """
 filegroup(
@ -56,3 +57,24 @@ def tool_deps():
        url = "https://github.com/sourcegraph/src-cli/releases/download/{0}/src-cli_{0}_darwin_arm64.tar.gz".format(SRC_CLI_VERSION),
    )

+    # universal-ctags #
+    http_file(
+        name = "universal-ctags-darwin-x86_64",
+        sha256 = "b69501d497b62021e8438e840e0bea62fdbe91d60cf8375c388f2736cd58a1bf",
+        url = "https://storage.googleapis.com/universal_ctags/x86_64-darwin/bin/universal-ctags-{0}".format(CTAGS_VERSION),
+        executable = True,
+    )
+
+    http_file(
+        name = "universal-ctags-darwin-arm64",
+        sha256 = "51b3b7ea296455e00fc5a7aafea49bb89551e81770b3728c97a04a5614fde8c5",
+        url = "https://storage.googleapis.com/universal_ctags/aarch64-darwin/bin/universal-ctags-{0}".format(CTAGS_VERSION),
+        executable = True,
+    )
+
+    http_file(
+        name = "universal-ctags-linux-amd64",
+        sha256 = "1d349d15736a30c9cc18c1fd9efbfc6081fb59125d799b84cef6b34c735fa28a",
+        url = "https://storage.googleapis.com/universal_ctags/x86_64-linux/bin/universal-ctags-{0}".format(CTAGS_VERSION),
+        executable = True,
+    )
--- a/dev/tools/BUILD.bazel
+++ b/dev/tools/BUILD.bazel
@ -17,3 +17,13 @@ sh_binary(
    }),
    visibility = ["//visibility:public"],
 )
+
+sh_binary(
+    name = "universal-ctags",
+    srcs = select({
+        "@bazel_tools//src/conditions:darwin_x86_64": ["@universal-ctags-darwin-amd64//file:downloaded"],
+        "@bazel_tools//src/conditions:darwin_arm64": ["@universal-ctags-darwin-arm64//file:downloaded"],
+        "@bazel_tools//src/conditions:linux_x86_64": ["@universal-ctags-linux-amd64//file:downloaded"],
+    }),
+    visibility = ["//visibility:public"],
+)
--- a/dev/universal-ctags-dev
+++ b/dev/universal-ctags-dev
@ -1,22 +1,8 @@
 #!/usr/bin/env bash

-# This script is a wrapper around `universal-ctags`.
-#
-# It checks if universal-ctags has been installed through ./dev/ctags-install.sh or falls back to a dockerized version.
+# This script is a wrapper around `universal-ctags` to download and run universal-ctags using bazel
 #
 # To use your own `universal-ctags` binary instead of this wrapper in your local dev server, use
 # `CTAGS_COMMAND=path/to/ctags sg start`.

-root="$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null
-TARGET=$("$root/dev/ctags-install.sh" which)
-
-if [ ! -f "${TARGET}" ]; then
-  exec docker run --rm -i \
-    -a stdin -a stdout -a stderr \
-    --user guest \
-    --name=universal-ctags-$$ \
-    --entrypoint /usr/local/bin/universal-ctags \
-    ctags "$@"
-else
-  ${TARGET} "$@"
-fi
+bazel run //dev/tools:universal-ctags -- "$@"
--- a/flake.lock
+++ b/flake.lock
@ -5,11 +5,11 @@
        "systems": "systems"
      },
      "locked": {
-        "lastModified": 1681202837,
-        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
+        "lastModified": 1689068808,
+        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
+        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
        "type": "github"
      },
      "original": {
@ -28,9 +28,10 @@
        "type": "github"
      },
      "original": {
-        "id": "nixpkgs",
-        "ref": "nixos-unstable",
-        "type": "indirect"
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "e78d25df6f1036b3fa76750ed4603dd9d5fe90fc",
+        "type": "github"
      }
    },
    "root": {
--- a/flake.nix
+++ b/flake.nix
@ -2,30 +2,35 @@
  description = "The Sourcegraph developer environment & packages Nix Flake";

  inputs = {
-    nixpkgs.url = "nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/e78d25df6f1036b3fa76750ed4603dd9d5fe90fc";
    flake-utils.url = "github:numtide/flake-utils";
  };

  outputs = { self, nixpkgs, flake-utils }:
+    let
+      xcompileTargets = with nixpkgs.lib.systems.examples; {
+        "aarch64-darwin" = nixpkgs.legacyPackages.aarch64-darwin.pkgsx86_64Darwin;
+        "x86_64-darwin" = import nixpkgs { system = "x86_64-darwin"; crossSystem = aarch64-darwin; };
+      };
+      inherit (import ./dev/nix/util.nix { inherit (nixpkgs) lib; }) xcompilify;
+    in
    flake-utils.lib.eachDefaultSystem
      (system:
        let
          pkgs = nixpkgs.legacyPackages.${system};
          pkgs' = import nixpkgs { inherit system; overlays = builtins.attrValues self.overlays; };
+          pkgsX = xcompileTargets.${system} or null;
        in
        {
-          # We set legacyPackages to our custom static binaries so command
-          # like "nix build .#p4-fusion" work.
          legacyPackages = pkgs';

-          packages = {
-            ctags = pkgs.callPackage ./dev/nix/ctags.nix { };
-            comby = pkgs.callPackage ./dev/nix/comby.nix { };
+          packages = xcompilify { inherit pkgs pkgsX; }
+            (pkgs: {
+              ctags = pkgs.callPackage ./dev/nix/ctags.nix { };
+              comby = pkgs.callPackage ./dev/nix/comby.nix { };
+              p4-fusion = pkgs.callPackage ./dev/nix/p4-fusion.nix { };
+            }) // {
            nodejs-16_x = pkgs.callPackage ./dev/nix/nodejs.nix { };
-          }
-          # so we don't get `packages.aarch64-linux.p4-fusion` in nix `flake show` output
-          // pkgs.lib.optionalAttrs (pkgs.targetPlatform.system != "aarch64-linux") {
-            p4-fusion = pkgs.callPackage ./dev/nix/p4-fusion.nix { };
          };

          # We use pkgs (not pkgs') intentionally to avoid doing extra work of
--- a/sg.config.yaml
+++ b/sg.config.yaml
@ -239,7 +239,6 @@ commands:
        export GCFLAGS='-N -l'
      fi

-      ./cmd/symbols/build-ctags.sh &&
      go build -gcflags="$GCFLAGS" -o .bin/symbols github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols
    checkBinary: .bin/symbols
    env:
@ -927,7 +926,6 @@ bazelCommands:
    env:
      ENTERPRISE: 1
  symbols:
-    # TODO build ctags thing
    target: //enterprise/cmd/symbols
    checkBinary: .bin/symbols
    env: