appliance: helm-compare utility (#62181)

Included README in the new package tells the story.

internal/appliance/dev/compare-helm/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "compare-helm_lib",
+    srcs = ["compare-helm.go"],
+    importpath = "github.com/sourcegraph/sourcegraph/internal/appliance/dev/compare-helm",
+    visibility = ["//visibility:private"],
+    deps = [
+        "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
+        "@io_k8s_apimachinery//pkg/util/yaml",
+        "@io_k8s_sigs_yaml//:yaml",
+    ],
+)
+
+go_binary(
+    name = "compare-helm",
+    embed = [":compare-helm_lib"],
+    visibility = ["//:__subpackages__"],
+)

internal/appliance/dev/compare-helm/README.md

@@ -0,0 +1,52 @@
+# compare-helm
+
+This is a utility with a very specific, and possibly short-lived, use-case. It
+prints a diff of appliance golden fixtures with resources from the templated
+Sourcegraph helm chart. "Appliance golden fixtures" are lists of yaml-serialized
+resources generated by the appliance (an in-development Kubernetes operator for
+Sourcegraph) in response to certain configuration inputs.
+
+## Usage
+
+Example usage:
+
+```
+go run ./internal/appliance/dev/compare-helm \
+  -component blobstore \
+  -golden-file internal/appliance/testdata/golden-fixtures/blobstore-default.yaml
+```
+
+Flags:
+
+- `component`: selects a subset of helm-templated resources for comparison.
+  Specifically, the "app.kubernetes.io/component" label is used. This appears to
+  correspond 1:1 with Sourcegraph services.
+- `golden-file`: path to a golden appliance fixture in this repo.
+- `helm-template-extra-args`: Optional extra arguments to pass to `helm
+template`. Useful for comparing helm value permutations to different golden
+  fixtures derived from different config inputs. Example: "--set
+  repoUpdater.serviceAccount.create=true".
+- `deploy-sourcegraph-helm-path`: path to a checkout of deploy-sourcegraph-helm.
+  This is needed unless you are running this command from the root of this repo,
+  and deploy-sourcegraph-helm is a sibling directory of your working directory.
+- `no-color`: optional, but needed if your version of `diff` doesn't support the
+  `--color=auto` option. GNU diff supports this, and can be installed on Mac
+  with `brew install diffutils`.
+
+## Interpreting the output
+
+Negative (red) diff is text that was output by helm but not by the golden file,
+and green (positive) diff is text that was output by the golden file but not by
+helm.
+
+In order to assist in figuring out which resource diff hunks correspond to, `#
+<helm|golden> $kind/$name` is printed at the top of each document. Documents are
+separated by the standard yaml multi-doc separator `---`.
+
+Resources that appear entirely in red are output by helm but not by the
+appliance, and vice-versa for resources that appear entirely in green.
+
+The order of resources in golden fixtures, and the order of keys in a given yaml
+document, shouldn't matter. This utility normalizes yaml documents (by
+unmarhsalling and re-marshalling), and also sorts the golden fixture into the
+same order as the helm output (by `{kind, metadata.name}`).

internal/appliance/dev/compare-helm/compare-helm.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	k8syamlapi "k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/yaml"
+)
+
+func main() {
+	helmRepoRoot := flag.String("deploy-sourcegraph-helm-path", filepath.Join("..", "deploy-sourcegraph-helm"), "Path to deploy-sourcegraph-helm repository checkout.")
+	helmTemplateExtraArgs := flag.String("helm-template-extra-args", "", "extra args to pass to `helm template`")
+	component := flag.String("component", "", "Which SG service to target.")
+	goldenFile := flag.String("golden-file", "", "Which golden fixture to compare.")
+	noColor := flag.Bool("no-color", false, "Do not try to produce diffs in color. This is necessary for non-GNU diff users.")
+	flag.Parse()
+
+	if *component == "" {
+		fatal("must pass -component")
+	}
+	if *goldenFile == "" {
+		fatal("must pass -golden-file")
+	}
+
+	helmObjs := parseHelmResources(*helmTemplateExtraArgs, *helmRepoRoot, *component)
+
+	goldenContent, err := os.ReadFile(*goldenFile)
+	must(err)
+	var goldenResources goldenResources
+	must(k8syamlapi.Unmarshal(goldenContent, &goldenResources))
+
+	tmpDir, err := os.MkdirTemp("", "compare-helm-")
+	must(err)
+	defer os.RemoveAll(tmpDir)
+	sortedGoldenPath := filepath.Join(tmpDir, "sorted-goldens.yaml")
+	helmTemplateOutputPath := filepath.Join(tmpDir, "sorted-helm-template.yaml")
+
+	sortedHelmResourceFile, err := openForWriting(helmTemplateOutputPath)
+	must(err)
+	sortedGoldenFile, err := openForWriting(sortedGoldenPath)
+	must(err)
+
+	// Write all helm and golden objects to their respective files for diffing.
+	// The order of these objects (by {kind, metadata.name}) should match, so
+	// that the diff has a chance of making sense.
+	// The key order within each object should be normalized too, since
+	// semantically we don't want that to influence the diff. We achieve this by
+	// unmarshalling and re-marshalling each object.
+	for _, helmObj := range helmObjs {
+		fmt.Fprintln(sortedHelmResourceFile, "---")
+		fmt.Fprintln(sortedGoldenFile, "---")
+
+		fmt.Fprintf(sortedHelmResourceFile, "# helm: %s/%s\n", helmObj.GetKind(), helmObj.GetName())
+		helmObjBytes, err := yaml.Marshal(helmObj)
+		must(err)
+		_, err = sortedHelmResourceFile.Write(helmObjBytes)
+		must(err)
+
+		// find corresponding golden object
+		for i, goldenObj := range goldenResources.Resources {
+			if helmObj.GetName() == goldenObj.GetName() &&
+				helmObj.GetKind() == goldenObj.GetKind() {
+
+				fmt.Fprintf(sortedGoldenFile, "# golden: %s/%s\n", helmObj.GetKind(), helmObj.GetName())
+				goldenBytes, err := yaml.Marshal(goldenObj)
+				must(err)
+				_, err = sortedGoldenFile.Write(goldenBytes)
+				must(err)
+
+				// remove the golden object so that only unmatched resources
+				// remain
+				goldenResources.Resources = append(goldenResources.Resources[:i], goldenResources.Resources[i+1:]...)
+
+				break
+			}
+		}
+	}
+
+	// Print any golden resources that didn't correspond to any helm resources,
+	// so that they appear in the diff.
+	for _, unmatchedGolden := range goldenResources.Resources {
+		fmt.Fprintln(sortedGoldenFile, "---")
+		fmt.Fprintf(sortedGoldenFile, "# golden: %s/%s\n", unmatchedGolden.GetKind(), unmatchedGolden.GetName())
+		goldenBytes, err := yaml.Marshal(unmatchedGolden)
+		must(err)
+		_, err = sortedGoldenFile.Write(goldenBytes)
+		must(err)
+	}
+
+	must(sortedHelmResourceFile.Close())
+	must(sortedGoldenFile.Close())
+
+	var diffCmdArgs []string
+	if !*noColor {
+		diffCmdArgs = append(diffCmdArgs, "--color=auto")
+	}
+	diffCmdArgs = append(diffCmdArgs, helmTemplateOutputPath, sortedGoldenPath)
+	diffCmd := exec.Command("diff", diffCmdArgs...)
+	diffCmd.Stdout = os.Stdout
+	diffCmd.Stderr = os.Stderr
+	must(diffCmd.Run())
+}
+
+func parseHelmResources(helmTemplateExtraArgs, helmRepoRoot, component string) []*unstructured.Unstructured {
+	helmShellCmd := "helm template . " + helmTemplateExtraArgs
+	helmCmd := exec.Command("sh", "-c", helmShellCmd)
+	helmCmd.Dir = filepath.Join(helmRepoRoot, "charts", "sourcegraph")
+
+	var helmStdout bytes.Buffer
+	helmCmd.Stdout = &helmStdout
+	helmCmd.Stderr = os.Stderr
+
+	must(helmCmd.Run())
+
+	var helmObjs []*unstructured.Unstructured
+	multiDocYAMLReader := k8syamlapi.NewYAMLReader(bufio.NewReader(&helmStdout))
+	for {
+		yamlDoc, err := multiDocYAMLReader.Read()
+		if err == io.EOF {
+			break
+		}
+		must(err)
+		jsonDoc, err := k8syamlapi.ToJSON(yamlDoc)
+		must(err)
+		obj, _, err := unstructured.UnstructuredJSONScheme.Decode(jsonDoc, nil, nil)
+		must(err)
+
+		k8sObj := obj.(*unstructured.Unstructured)
+		if k8sObj.GetLabels()["app.kubernetes.io/component"] == component {
+			helmObjs = append(helmObjs, k8sObj)
+		}
+	}
+	return helmObjs
+}
+
+// A shame to dup this
+type goldenResources struct {
+	Resources []*unstructured.Unstructured `json:"resources"`
+}
+
+func openForWriting(pathname string) (*os.File, error) {
+	return os.OpenFile(pathname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+}
+
+func must(err error) {
+	if err != nil {
+		fatal(err.Error())
+	}
+}
+
+func fatal(msg string) {
+	fmt.Fprintln(os.Stderr, msg)
+	os.Exit(1)
+}