Revert "symbols: replace pcre lib with a Go function (fixup) (#25401)" (#25410)

This reverts commit 853892694f.

.gitignore

@@ -113,6 +113,10 @@ dll-bundle
 # Extensions
 /client/extension-api/dist
 
+# Symbols service: PCRE extension to sqlite3
+libsqlite3-pcre.dylib
+libsqlite3-pcre.so
+
 # Direnv
 .envrc
 .direnv

cmd/server/Dockerfile

@@ -1,3 +1,8 @@
+FROM sourcegraph/alpine-3.12:107969_2021-09-10_80f5edc@sha256:ce1ba2f16ec56e5e8007da53e0e6449bc0fa1fe1f972bffbc33dea1ae410b86d as libsqlite3-pcre
+
+COPY libsqlite3-pcre-install-alpine.sh /libsqlite3-pcre-install-alpine.sh
+RUN /libsqlite3-pcre-install-alpine.sh
+
 # Install p4 CLI (keep this up to date with cmd/gitserver/Dockerfile)
 FROM sourcegraph/alpine-3.12:107969_2021-09-10_80f5edc@sha256:ce1ba2f16ec56e5e8007da53e0e6449bc0fa1fe1f972bffbc33dea1ae410b86d AS p4cli
 
@@ -92,6 +97,9 @@ RUN set -ex && \
 # hadolint ignore=DL3022
 COPY --from=sourcegraph/grafana:server /usr/share/grafana /usr/share/grafana
 
+# hadolint ignore=DL3022
+COPY --from=libsqlite3-pcre /sqlite3-pcre/pcre.so /libsqlite3-pcre.so
+ENV LIBSQLITE3_PCRE /libsqlite3-pcre.so
 COPY . /
 
 # hadolint ignore=DL3022

cmd/server/build.sh

@@ -76,6 +76,7 @@ parallel_run go_build {} ::: "${PACKAGES[@]}"
 
 echo "--- ctags"
 cp -a ./cmd/symbols/ctags-install-alpine.sh "$OUTPUT"
+cp -a ./dev/libsqlite3-pcre/install-alpine.sh "$OUTPUT/libsqlite3-pcre-install-alpine.sh"
 
 echo "--- monitoring generation"
 # For code generation we need to match the local machine so we can run the generator

cmd/symbols/Dockerfile

@@ -6,6 +6,13 @@ USER root
 COPY ctags-install-alpine.sh /ctags-install-alpine.sh
 RUN /ctags-install-alpine.sh
 
+FROM sourcegraph/alpine-3.12:107969_2021-09-10_80f5edc@sha256:ce1ba2f16ec56e5e8007da53e0e6449bc0fa1fe1f972bffbc33dea1ae410b86d as libsqlite3-pcre
+# hadolint ignore=DL3002
+USER root
+
+COPY libsqlite3-pcre-install-alpine.sh /libsqlite3-pcre-install-alpine.sh
+RUN /libsqlite3-pcre-install-alpine.sh
+
 FROM sourcegraph/alpine-3.12:107969_2021-09-10_80f5edc@sha256:ce1ba2f16ec56e5e8007da53e0e6449bc0fa1fe1f972bffbc33dea1ae410b86d AS symbols
 
 # TODO(security): This container should not run as root!
@@ -29,6 +36,12 @@ RUN apk add --no-cache bind-tools ca-certificates mailcap tini
 COPY ctags-install-alpine.sh /ctags-install-alpine.sh
 RUN /ctags-install-alpine.sh
 
+# hadolint ignore=DL3022
+COPY --from=libsqlite3-pcre /sqlite3-pcre/pcre.so /libsqlite3-pcre.so
+ENV LIBSQLITE3_PCRE /libsqlite3-pcre.so
+# hadolint ignore=DL3018
+RUN apk --no-cache add pcre-dev
+
 ENV CACHE_DIR=/mnt/cache/symbols
 RUN mkdir -p ${CACHE_DIR}
 EXPOSE 3184

cmd/symbols/build.sh

@@ -12,6 +12,7 @@ cleanup() {
 trap cleanup EXIT
 
 cp -a ./cmd/symbols/ctags-install-alpine.sh "$OUTPUT"
+cp -a ./dev/libsqlite3-pcre/install-alpine.sh "$OUTPUT/libsqlite3-pcre-install-alpine.sh"
 
 # Build go binary into $OUTPUT
 ./cmd/symbols/go-build.sh "$OUTPUT"

cmd/symbols/go-build.sh

@@ -12,7 +12,9 @@ OUTPUT="${1:?no output path provided}"
 export GO111MODULE=on
 export GOARCH=amd64
 export GOOS=linux
-export CGO_ENABLED=0
+
+# Get additional build args
+. ./dev/libsqlite3-pcre/go-build-args.sh
 
 echo "--- go build"
 pkg="github.com/sourcegraph/sourcegraph/cmd/symbols"

cmd/symbols/internal/sqliteutil/register.go

@@ -0,0 +1,50 @@
+package sqliteutil
+
+import (
+	"database/sql"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"runtime"
+	"strings"
+
+	"github.com/cockroachdb/errors"
+	"github.com/mattn/go-sqlite3"
+
+	"github.com/sourcegraph/sourcegraph/internal/env"
+)
+
+var libSqlite3Pcre = env.Get("LIBSQLITE3_PCRE", "", "path to the libsqlite3-pcre library")
+
+// MustRegisterSqlite3WithPcre registers a sqlite3 driver with PCRE support and
+// panics if it can't.
+func MustRegisterSqlite3WithPcre() {
+	if libSqlite3Pcre == "" {
+		env.PrintHelp()
+		log.Fatal("can't find the libsqlite3-pcre library because LIBSQLITE3_PCRE was not set")
+	}
+	sql.Register("sqlite3_with_pcre", &sqlite3.SQLiteDriver{Extensions: []string{libSqlite3Pcre}})
+}
+
+// SetLocalLibpath sets the path to the LIBSQLITE3_PCRE shared library. This should
+// be called only in test environments. Production environments must require that
+// the envvar be set explicitly.
+func SetLocalLibpath() {
+	if libSqlite3Pcre != "" {
+		return
+	}
+
+	repositoryRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
+	if err != nil {
+		panic("can't find the libsqlite3-pcre library because LIBSQLITE3_PCRE was not set and you're not in the git repository, which is where the library is expected to be.")
+	}
+	if runtime.GOOS == "darwin" {
+		libSqlite3Pcre = path.Join(strings.TrimSpace(string(repositoryRoot)), "libsqlite3-pcre.dylib")
+	} else {
+		libSqlite3Pcre = path.Join(strings.TrimSpace(string(repositoryRoot)), "libsqlite3-pcre.so")
+	}
+	if _, err := os.Stat(libSqlite3Pcre); os.IsNotExist(err) {
+		panic(errors.Errorf("can't find the libsqlite3-pcre library because LIBSQLITE3_PCRE was not set and %s doesn't exist at the root of the repository - try building it with `./dev/libsqlite3-pcre/build.sh`", libSqlite3Pcre))
+	}
+}

cmd/symbols/internal/symbols/search.go

@@ -2,17 +2,13 @@ package symbols
 
 import (
 	"context"
-	"database/sql"
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"regexp"
 	"regexp/syntax"
 	"strings"
 	"time"
 
-	"github.com/mattn/go-sqlite3"
-
 	"github.com/sourcegraph/sourcegraph/internal/api"
 
 	"github.com/inconshreveable/log15"
@@ -28,15 +24,6 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/search/result"
 )
 
-func init() {
-	sql.Register("sqlite3_with_regexp",
-		&sqlite3.SQLiteDriver{
-			ConnectHook: func(conn *sqlite3.SQLiteConn) error {
-				return conn.RegisterFunc("REGEXP", regexp.MatchString, true)
-			},
-		})
-}
-
 // maxFileSize is the limit on file size in bytes. Only files smaller than this are processed.
 const maxFileSize = 1 << 19 // 512KB
 
@@ -95,7 +82,7 @@ func (s *Service) search(ctx context.Context, args protocol.SearchArgs) (*result
 	if err != nil {
 		return nil, err
 	}
-	db, err := sqlx.Open("sqlite3_with_regexp", dbFile)
+	db, err := sqlx.Open("sqlite3_with_pcre", dbFile)
 	if err != nil {
 		return nil, err
 	}
@@ -288,7 +275,7 @@ func symbolInDBToSymbol(symbolInDB symbolInDB) result.Symbol {
 // writeAllSymbolsToNewDB fetches the repo@commit from gitserver, parses all the
 // symbols, and writes them to the blank database file `dbFile`.
 func (s *Service) writeAllSymbolsToNewDB(ctx context.Context, dbFile string, repoName api.RepoName, commitID api.CommitID) (err error) {
-	db, err := sqlx.Open("sqlite3_with_regexp", dbFile)
+	db, err := sqlx.Open("sqlite3_with_pcre", dbFile)
 	if err != nil {
 		return err
 	}

cmd/symbols/internal/symbols/search_test.go

@@ -10,10 +10,13 @@ import (
 	"github.com/inconshreveable/log15"
 
 	"github.com/sourcegraph/sourcegraph/cmd/symbols/internal/protocol"
+	"github.com/sourcegraph/sourcegraph/cmd/symbols/internal/sqliteutil"
 	"github.com/sourcegraph/sourcegraph/internal/testutil"
 )
 
 func BenchmarkSearch(b *testing.B) {
+	sqliteutil.MustRegisterSqlite3WithPcre()
+
 	log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlError, log15.Root().GetHandler()))
 
 	service := Service{

cmd/symbols/internal/symbols/service_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/sourcegraph/go-ctags"
 
+	"github.com/sourcegraph/sourcegraph/cmd/symbols/internal/sqliteutil"
 	"github.com/sourcegraph/sourcegraph/internal/api"
 	"github.com/sourcegraph/sourcegraph/internal/httpcli"
 	"github.com/sourcegraph/sourcegraph/internal/search"
@@ -19,6 +20,10 @@ import (
 	symbolsclient "github.com/sourcegraph/sourcegraph/internal/symbols"
 )
 
+func init() {
+	sqliteutil.SetLocalLibpath()
+}
+
 func TestIsLiteralEquality(t *testing.T) {
 	type TestCase struct {
 		Regex       string
@@ -57,6 +62,8 @@ func TestIsLiteralEquality(t *testing.T) {
 }
 
 func TestService(t *testing.T) {
+	sqliteutil.MustRegisterSqlite3WithPcre()
+
 	tmpDir, err := os.MkdirTemp("", "")
 	if err != nil {
 		t.Fatal(err)

cmd/symbols/main.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/inconshreveable/log15"
 
+	"github.com/sourcegraph/sourcegraph/cmd/symbols/internal/sqliteutil"
 	"github.com/sourcegraph/sourcegraph/cmd/symbols/internal/symbols"
 	"github.com/sourcegraph/sourcegraph/internal/api"
 	"github.com/sourcegraph/sourcegraph/internal/conf"
@@ -47,6 +48,8 @@ func main() {
 	sentry.Init()
 	trace.Init()
 
+	sqliteutil.MustRegisterSqlite3WithPcre()
+
 	// Ready immediately
 	ready := make(chan struct{})
 	close(ready)

dev/ci/go-test.sh

@@ -2,6 +2,10 @@
 
 set -e
 
+# For symbol tests
+echo "--- build libsqlite"
+./dev/libsqlite3-pcre/build.sh
+
 # For searcher
 echo "--- comby install"
 ./dev/comby-install-or-upgrade.sh

dev/libsqlite3-pcre/build.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# This script will ensure that the libsqlite3-pcre dynamic library exists in the root of this
+# repository (either libsqlite3-pcre.dylib for Darwin or libsqlite3-pcre.so for linux). This
+# script is used by run the symbol service locally, which compiles against the shared library.
+#
+# Invocation:
+# - `./libsqlite3-pcre/build.sh`         : build the library
+# - `./libsqlite3-pcre/build.sh libpath` : output its path
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.."
+set -eu
+
+OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
+cleanup() {
+  rm -rf "$OUTPUT"
+}
+trap cleanup EXIT
+
+# Print the absolute path of the sqlite3 shared library for this platform,
+# or terminate with an error.
+function libpath() {
+  case "$OSTYPE" in
+    darwin*)
+      echo "$PWD/libsqlite3-pcre.dylib"
+      ;;
+
+    linux*)
+      echo "$PWD/libsqlite3-pcre.so"
+      ;;
+
+    *)
+      echo "Unknown platform $OSTYPE"
+      exit 1
+      ;;
+  esac
+}
+
+function build() {
+  local libsqlite_path
+  libsqlite_path=$(libpath)
+
+  if [[ -f "$libsqlite_path" ]]; then
+    # Already exists
+    exit 0
+  fi
+
+  echo "--- libsqlite3-pcre build"
+
+  if ! command -v pkg-config >/dev/null 2>&1 || ! command -v pkg-config --cflags sqlite3 libpcre >/dev/null 2>&1; then
+    echo "Missing sqlite dependencies."
+    case "$OSTYPE" in
+      darwin*)
+        echo "Install them by running 'brew install pkg-config sqlite pcre FiloSottile/musl-cross/musl-cross'"
+        ;;
+
+      linux*)
+        echo "Install them by running 'apt-get install libpcre3-dev libsqlite3-dev pkg-config musl-tools'"
+        ;;
+
+      *)
+        echo "See the local development documentation: https://github.com/sourcegraph/sourcegraph/blob/main/doc/dev/local_development.md#step-2-install-dependencies"
+        ;;
+    esac
+
+    exit 1
+  fi
+
+  echo "--- $libsqlite_path build"
+  curl -fsSL https://codeload.github.com/ralight/sqlite3-pcre/tar.gz/c98da412b431edb4db22d3245c99e6c198d49f7a | tar -C "$OUTPUT" -xzvf - --strip 1
+  cd "$OUTPUT"
+
+  case "$OSTYPE" in
+    darwin*)
+      # pkg-config spits out multiple arguments and must not be quoted.
+      # shellcheck disable=SC2046
+      gcc -fno-common -dynamiclib pcre.c -o "$libsqlite_path" $(pkg-config --cflags sqlite3 libpcre) $(pkg-config --libs libpcre sqlite3) -fPIC
+      exit 0
+      ;;
+
+    linux*)
+      # pkg-config spits out multiple arguments and must not be quoted.
+      # shellcheck disable=SC2046
+      gcc -shared -o "$libsqlite_path" $(pkg-config --cflags sqlite3 libpcre) -fPIC -W -Werror pcre.c $(pkg-config --libs libpcre sqlite3) -Wl,-z,defs
+      exit 0
+      ;;
+
+    *)
+      echo "See the local development documentation: https://github.com/sourcegraph/sourcegraph/blob/main/doc/dev/local_development.md#step-2-install-dependencies"
+      echo "Unknown platform $OSTYPE"
+      exit 1
+      ;;
+  esac
+}
+
+# Execute $1 (build by default)
+"${1:-build}"

dev/libsqlite3-pcre/go-build-args.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# This script exports variables required to link to the libsqlite3-pcre library.
+# This should be sourced prior to the go build command for the binaries that
+# require sqlite as a dependency.
+#
+# Usage: `. ./dev/libsqlite3-pcre/go-build-args.sh`
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.."
+set -eu
+
+# Set default empty GOOS
+GOOS="${GOOS:-''}"
+
+if [[ "$GOOS" != "linux" ]]; then
+  exit 0
+fi
+
+case "$OSTYPE" in
+  darwin*)
+    muslGcc="x86_64-linux-musl-gcc"
+    if ! command -v "$muslGcc" >/dev/null 2>&1; then
+      echo "Couldn't find musl C compiler $muslGcc. Run 'brew install FiloSottile/musl-cross/musl-cross'."
+      exit 1
+    fi
+    ;;
+
+  linux*)
+    muslGcc="musl-gcc"
+    if ! command -v "$muslGcc" >/dev/null 2>&1; then
+      echo "Couldn't find musl C compiler $muslGcc. Install the musl-tools package (e.g. on Ubuntu, run 'apt-get install musl-tools')."
+      exit 1
+    fi
+    ;;
+
+  *)
+    echo "Unknown platform $OSTYPE"
+    exit 1
+    ;;
+esac
+
+export CC="$muslGcc"
+export CGO_ENABLED=1

dev/libsqlite3-pcre/install-alpine.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# This script installs libsqlite3-pcre within an alpine container.
+
+set -eux
+
+# Commit hash of github.com/ralight/sqlite3-pcre
+SQLITE3_PCRE_VERSION=c98da412b431edb4db22d3245c99e6c198d49f7a
+
+apk --no-cache add \
+  --virtual build-deps \
+  curl \
+  gcc \
+  git \
+  libc-dev \
+  make \
+  pcre-dev \
+  sqlite-dev
+
+# Installation
+mkdir /sqlite3-pcre
+curl -fsSL "https://codeload.github.com/ralight/sqlite3-pcre/tar.gz/$SQLITE3_PCRE_VERSION" | tar -C /sqlite3-pcre -xzvf - --strip 1
+cd /sqlite3-pcre
+make
+
+# Cleanup
+apk --no-cache --purge del build-deps

dev/nix/shell-hook.sh

@@ -10,6 +10,9 @@
 
 pushd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null || exit
 
+# TODO build with nix
+NIX_ENFORCE_PURITY=0 ../libsqlite3-pcre/build.sh
+
 . ./start-postgres.sh
 . ./start-redis.sh
 

dev/start.sh

@@ -178,6 +178,11 @@ fi
 export NODE_ENV=development
 export NODE_OPTIONS="--max_old_space_size=4096"
 
+# Ensure SQLite for symbols is built
+./dev/libsqlite3-pcre/build.sh
+LIBSQLITE3_PCRE="$(./dev/libsqlite3-pcre/build.sh libpath)"
+export LIBSQLITE3_PCRE
+
 # Increase ulimit (not needed on Windows/WSL)
 # shellcheck disable=SC2015
 type ulimit >/dev/null && ulimit -n 10000 || true

doc/dev/getting-started/quickstart_1_install_dependencies.md

@@ -22,7 +22,7 @@ Below are instructions to install these dependencies:
 
 > NOTE: You can choose to install and run Redis and/or PostgreSQL **with or without Docker**. The following instructions will describe both options.
 > What's the better option?
->
+> 
 > - Running within a container provides some advantages such as storing the data separately from the container, you do not need to run it as a system service and its easy to use different database versions or multiple databases.
 > - Running as system services might yield better performance, especially on macOS.
 > - No matter which option you choose, docker is required because the development server starts additional docker containers.
@@ -41,13 +41,11 @@ Below are instructions to install these dependencies:
 3.  Install Go, Yarn, Git, Comby, SQLite tools, and jq with the following command:
 
     ```
-    brew install go yarn git gnu-sed comby pcre sqlite jq
+    brew install go yarn git gnu-sed comby sqlite pcre FiloSottile/musl-cross/musl-cross jq
     ```
 
 4. Choose to run Postgres and Redis manually (Option a.) or via Docker (Option b.)
 
-    a. (without docker) Install PostgreSQL and Redis
-
     a. **Without Docker**
 
       1. Install PostgreSQL and Redis with the following commands:
@@ -161,7 +159,7 @@ Below are instructions to install these dependencies:
 3. Install dependencies:
 
     ```
-    sudo apt install -y make git-all libpcre3-dev libsqlite3-dev pkg-config golang-go docker-ce docker-ce-cli containerd.io yarn jq libnss3-tools
+    sudo apt install -y make git-all libpcre3-dev libsqlite3-dev pkg-config golang-go musl-tools docker-ce docker-ce-cli containerd.io yarn jq libnss3-tools
 
     # Install comby
     curl -L https://github.com/comby-tools/comby/releases/download/0.11.3/comby-0.11.3-x86_64-linux.tar.gz | tar xvz

enterprise/dev/ci/internal/ci/changed.go

@@ -61,6 +61,7 @@ func (c ChangedFiles) affectsClient() bool {
 var ignoredRootFiles = []string{
 	"jest.config.base.js",
 	"graphql-schema-linter.config.js",
+	"libsqlite3-pcre.dylib",
 	".mocharc.js",
 	"go.mod",
 	"LICENSE",

sg.config.yaml

@@ -222,13 +222,9 @@ commands:
       - cmd/query-runner
 
   symbols:
-    cmd: .bin/symbols
+    cmd: env LIBSQLITE3_PCRE=$(./dev/libsqlite3-pcre/build.sh libpath) .bin/symbols
     install: |
-      # Remove old pcre libs that might still be lying around.
-      # TODO delete these two lines after 2021-10-24 (1 month after removal of pcre).
-      rm -f libsqlite3-pcre.dylib || true
-      rm -f libsqlite3-pcre.so || true
-
+      ./dev/libsqlite3-pcre/build.sh &&
       ./cmd/symbols/build-ctags.sh &&
       go build -o .bin/symbols github.com/sourcegraph/sourcegraph/cmd/symbols
     checkBinary: .bin/symbols

shell.nix

@@ -63,6 +63,7 @@ in pkgs.mkShell {
     pkgs.parallel
 
     # cgo dependency for symbols. TODO build with nix?
+    pkgs.pcre
     pkgs.sqlite
     pkgs.pkg-config