bazel: fetch code-intel-extensions bundle as bazel repository (#59708)

Reimplement fetching & building code-intel-extension bundle as a bazel `http_archive` repository & refactored `js_run_binary`. This should allow the extensions bundle to be cached in the repository cache, instead of being affected by changing action environments

## Test plan

`bazel build //client/browser:code-intel-extensions` and `cd client/browser && pnpm run create-source-zip` run without error

WORKSPACE

@@ -433,3 +433,23 @@ gazelle_buf_dependencies()
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
+
+# keep revision up-to-date with client/browser/scripts/build-inline-extensions.js
+http_archive(
+    name = "sourcegraph_extensions_bundle",
+    add_prefix = "bundle",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+
+exports_files(["bundle"])
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"]
+)
+    """,
+    integrity = "sha256-Spx8LyM7k+dsGOlZ4TdAq+CNk5EzvYB/oxnY4zGpqPg=",
+    strip_prefix = "sourcegraph-extensions-bundles-5.0.1",
+    url = "https://github.com/sourcegraph/sourcegraph-extensions-bundles/archive/v5.0.1.zip",
+)

client/browser/BUILD.bazel

@@ -375,7 +375,6 @@ esbuild(
 build_code_intel_extensions(
     name = "code-intel-extensions",
     out = "build/extensions",
-    revision = "v5.0.1",
 )
 
 # Mirror `copyIntegrationAssets` from `client/browser/scripts/tasks.ts`

client/browser/scripts/build-inline-extensions.js

@@ -1,7 +1,8 @@
 const path = require('path')
 
-const { buildCodeIntelExtensions } = require('../../shared/dev/buildCodeIntelExtensions')
+const { fetchAndBuildCodeIntelExtensions } = require('../../shared/dev/buildCodeIntelExtensions')
 
 const pathToExtensionBundles = path.join(process.cwd(), 'build', 'extensions')
 
-buildCodeIntelExtensions({ pathToExtensionBundles, revision: 'v5.0.1' })
+// keep revision up-to-date with "sourcegraph_extensions_bundle" in WORKSPACE
+fetchAndBuildCodeIntelExtensions({ pathToExtensionBundles, revision: 'v5.0.1' })

client/shared/dev/buildCodeIntelExtensions.js

@@ -12,7 +12,7 @@ const bundlesRepoName = 'sourcegraph-extensions-bundles'
  * extensions bundles to the specified path. These bundles may be included into IDE/browser extensions bundles
  * in order not to require access to the extensions registry for the code intel features to work.
  */
-function buildCodeIntelExtensions({ pathToExtensionBundles, revision }) {
+function fetchAndBuildCodeIntelExtensions({ pathToExtensionBundles, revision }) {
   const pathToDistributionRevisionFile = path.join(pathToExtensionBundles, 'revision.txt')
 
   const currentRevision =
@@ -32,6 +32,7 @@ function buildCodeIntelExtensions({ pathToExtensionBundles, revision }) {
   if (result.code !== 0) {
     console.error('Curl command failed with exit code:', result.code)
     console.error('Error message:', result.stderr)
+    return
   }
 
   // when repo archive is unpacked the leading 'v' from tag is trimmed: v1.0.0.zip => sourcegraph-extensions-bundles-1.0.0
@@ -46,8 +47,22 @@ function buildCodeIntelExtensions({ pathToExtensionBundles, revision }) {
   // Remove bundles repo archive
   shelljs.exec(`rm ${revision}.zip`)
 
+  buildCodeIntelExtensions({
+    extensionBundlesSrc: bundlesRepoDirectoryName,
+    extensionBundlesDest: pathToExtensionBundles,
+    revision,
+  })
+
+  // Remove bundles repo directory and archive
+  shelljs.exec(`rm -rf ${bundlesRepoDirectoryName}`)
+
+  // Save sourcegraph-extensions-bundles revision not to refetch it on the next calls if the revision doesn't change
+  fs.writeFileSync(pathToDistributionRevisionFile, revision)
+}
+
+function buildCodeIntelExtensions({ extensionBundlesSrc, extensionBundlesDest }) {
   const codeIntelExtensionIds = [] // list of cloned code intel extension ids, e.g. [..., 'sourcegraph/typescript', ...]
-  const content = fs.readdirSync(path.join(bundlesRepoDirectoryName, 'bundles'), { withFileTypes: true })
+  const content = fs.readdirSync(path.join(extensionBundlesSrc, 'bundles'), { withFileTypes: true })
 
   for (const item of content) {
     if (!item.isDirectory()) {
@@ -56,7 +71,7 @@ function buildCodeIntelExtensions({ pathToExtensionBundles, revision }) {
 
     const extensionName = item.name // kebab-case extension name, e.g. 'sourcegraph-typescript'
 
-    const bundlePath = path.join(bundlesRepoDirectoryName, 'bundles', extensionName)
+    const bundlePath = path.join(extensionBundlesSrc, 'bundles', extensionName)
     const files = fs.readdirSync(bundlePath)
 
     if (['package.json', `${extensionName}.js`].some(file => !files.includes(file))) {
@@ -77,13 +92,13 @@ function buildCodeIntelExtensions({ pathToExtensionBundles, revision }) {
       continue
     }
 
-    shelljs.mkdir('-p', path.join(pathToExtensionBundles, extensionName))
+    shelljs.mkdir('-p', path.join(extensionBundlesDest, extensionName))
     shelljs.exec(
-      `cp ${path.join(bundlePath, 'package.json')} ${path.join(pathToExtensionBundles, extensionName, 'package.json')}`
+      `cp ${path.join(bundlePath, 'package.json')} ${path.join(extensionBundlesDest, extensionName, 'package.json')}`
     )
     shelljs.exec(
       `cp ${path.join(bundlePath, `${extensionName}.js`)} ${path.join(
-        pathToExtensionBundles,
+        extensionBundlesDest,
         extensionName,
         'extension.js'
       )}`
@@ -92,30 +107,25 @@ function buildCodeIntelExtensions({ pathToExtensionBundles, revision }) {
     codeIntelExtensionIds.push(extensionName.replace(/^sourcegraph-/, 'sourcegraph/'))
   }
 
-  // Remove bundles repo directory and archive
-  shelljs.exec(`rm -rf ${bundlesRepoDirectoryName}`)
-
-  // Save sourcegraph-extensions-bundles revision not to refetch it on the next calls if the revision doesn't change
-  fs.writeFileSync(pathToDistributionRevisionFile, revision)
-
   // Save extension IDs of the copied bundles
   fs.writeFileSync(path.join(process.cwd(), 'code-intel-extensions.json'), JSON.stringify(codeIntelExtensionIds))
 
   signale.success('Code intel extensions bundles successfully copied.')
 }
 
-module.exports = { buildCodeIntelExtensions }
+module.exports = { buildCodeIntelExtensions, fetchAndBuildCodeIntelExtensions }
 
 // Use this script in Bazel. Remove `module.exports` once the Bazel migration is completed.
 function main(args) {
   if (args.length !== 2) {
-    throw new Error('Usage: <revision> <outputPath>')
+    throw new Error('Usage: <inputPath> <outputPath>')
   }
 
-  const [revision, outputPath] = args
+  const [inputPath, outputPath] = args
   const output = path.join(process.cwd(), outputPath)
+  const input = path.join(process.env['JS_BINARY__EXECROOT'], inputPath)
 
-  buildCodeIntelExtensions({ pathToExtensionBundles: output, revision })
+  buildCodeIntelExtensions({ extensionBundlesSrc: input, extensionBundlesDest: output })
 }
 
 if (require.main === module) {

client/shared/dev/build_code_intel_extensions.bzl

@@ -2,13 +2,12 @@
 
 load("@aspect_rules_js//js:defs.bzl", "js_run_binary")
 
-def build_code_intel_extensions(name, out, revision):
+def build_code_intel_extensions(name, out):
     """ Download code-intel extension bundles from GitHub.
 
     Args:
         name: target name
         out: output revisions folder
-        revision: revision
     """
     js_run_binary(
         name = name,
@@ -17,12 +16,10 @@ def build_code_intel_extensions(name, out, revision):
         log_level = "info",
         silent_on_success = False,
         args = [
-            revision,
+            "$(execpath @sourcegraph_extensions_bundle//:bundle)",
             out,
         ],
-        tags = [
-            # We download static assets from GitHub.
-            "requires-network",
-        ],
+        srcs = ["@sourcegraph_extensions_bundle//:bundle"],
+        copy_srcs_to_bin = False,
         tool = "//client/shared/dev:build_code_intel_extensions",
     )

shell.nix

@@ -71,6 +71,7 @@ mkShell.override { stdenv = if hostPlatform.isMacOS then pkgs.clang11Stdenv else
   # The packages in the `buildInputs` list will be added to the PATH in our shell
   nativeBuildInputs = with pkgs; [
     bashInteractive
+    zip
 
     # nix language server.
     nil