Render and validate every app on PRs and pushes

.github/workflows/validate.yaml

@@ -0,0 +1,56 @@
+# Renders every Argo CD app the way Argo CD does and validates the result
+# (tools/ci/validate.py), so a broken manifest or a Renovate bump that no
+# longer renders shows up on the PR instead of in a failed sync.
+name: validate
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: validate-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Install helm, kustomize and kubeconform
+        env:
+          # Helm 4.2, like Argo CD 3.5.
+          # renovate: datasource=github-releases depName=helm/helm
+          HELM_VERSION: v4.2.1
+          # renovate: datasource=github-releases depName=kubernetes-sigs/kustomize extractVersion=^kustomize/(?<version>.+)$
+          KUSTOMIZE_VERSION: v5.8.3
+          # renovate: datasource=github-releases depName=yannh/kubeconform
+          KUBECONFORM_VERSION: v0.8.0
+        run: |
+          set -euo pipefail
+          cd "$RUNNER_TEMP"
+          curl -sSfLO "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz"
+          curl -sSfL "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz.sha256sum" | sha256sum -c -
+          k="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}"
+          curl -sSfLO "$k/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz"
+          curl -sSfL "$k/checksums.txt" | grep " kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz$" | sha256sum -c -
+          c="https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}"
+          curl -sSfLO "$c/kubeconform-linux-amd64.tar.gz"
+          curl -sSfL "$c/CHECKSUMS" | grep " kubeconform-linux-amd64.tar.gz$" | sha256sum -c -
+          mkdir -p bin
+          tar xzf "helm-${HELM_VERSION}-linux-amd64.tar.gz" -C bin --strip-components=1 linux-amd64/helm
+          tar xzf "kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" -C bin kustomize
+          tar xzf kubeconform-linux-amd64.tar.gz -C bin kubeconform
+          echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
+          pip install --quiet pyyaml==6.0.3
+
+      - name: Render and validate
+        run: python3 tools/ci/validate.py

renovate.json

@@ -1,7 +1,8 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "config:recommended"
+    "config:recommended",
+    "customManagers:githubActionsVersions"
   ],
   "argocd": {
     "managerFilePatterns": [

tools/ci/validate.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""Render every Argo CD app in this repo the way Argo CD does and validate
+the output with kubeconform, so a broken manifest or a chart bump that no
+longer renders fails the PR instead of the sync.
+
+Run from anywhere; needs helm, kustomize, kubeconform, git and PyYAML.
+The apps/ chart is rendered first, then every Application found in it and
+in the app folders. Sources:
+- a chart: helm template with the Application's values, value files and
+  parameters
+- a path in this repo: helm template (Chart.yaml), kustomize build
+  (kustomization.yaml) or the folder's YAML files (directory include and
+  exclude)
+- a path in another git repository: cloned at targetRevision, then the
+  same, with the Application's kustomize patches applied
+Every YAML file in the repo is also checked for duplicate keys, which
+YAML parsers resolve silently by keeping the last one.
+"""
+import fnmatch
+import os
+import pathlib
+import subprocess
+import sys
+import tempfile
+
+import yaml
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+OWN_REPOS = {"https://github.com/rubenhensen/k8scd.git", "https://github.com/rubenhensen/k8scd"}
+SKIP_DIRS = {".git", ".github", "nix-infra-machine", "docs", "tools", "backup-testing"}
+# Helm's view of the cluster; the JSON schemas only go up to 1.34.
+KUBE_VERSION = "1.36.0"
+SCHEMA_VERSION = "1.34.12"
+CRD_SCHEMAS = ("https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/"
+               "{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json")
+
+failures = []
+
+
+def run(cmd, **kw):
+    return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw).stdout
+
+
+class UniqueKeyLoader(yaml.SafeLoader):
+    pass
+
+
+def construct_mapping(loader, node, deep=False):
+    keys = set()
+    for key_node, _ in node.value:
+        key = loader.construct_object(key_node, deep=deep)
+        if key in keys:
+            raise yaml.constructor.ConstructorError(
+                None, None, f"duplicate key {key!r}", key_node.start_mark)
+        keys.add(key)
+    return loader.construct_mapping(node, deep)
+
+
+UniqueKeyLoader.add_constructor(
+    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
+
+
+def yaml_files():
+    for path in sorted(ROOT.rglob("*.y*ml")):
+        rel = path.relative_to(ROOT)
+        if rel.parts[0] in SKIP_DIRS or path.suffix not in (".yaml", ".yml"):
+            continue
+        yield path
+
+
+def check_duplicate_keys():
+    for path in yaml_files():
+        if path.parent.name == "templates":  # Helm templates aren't YAML yet
+            continue
+        try:
+            list(yaml.load_all(path.read_text(), Loader=UniqueKeyLoader))
+        except yaml.YAMLError as e:
+            failures.append(f"{path.relative_to(ROOT)}: {e}")
+
+
+def validate(label, manifests):
+    """kubeconform on rendered manifests; returns the parsed documents."""
+    try:
+        docs = [d for d in yaml.safe_load_all(manifests) if d]
+    except yaml.YAMLError:
+        docs = None  # kubeconform reports where
+    if docs is not None:
+        # Argo CD skips documents that aren't objects, like a values file
+        # next to the manifests.
+        objects = [d for d in docs if isinstance(d, dict) and "kind" in d]
+        if len(objects) < len(docs):
+            print(f"  {label}: skipping {len(docs) - len(objects)} document(s) without a kind")
+        manifests = yaml.safe_dump_all(objects, sort_keys=False)
+    result = subprocess.run(
+        ["kubeconform", "-summary", "-output", "text", "-strict",
+         "-kubernetes-version", SCHEMA_VERSION, "-ignore-missing-schemas",
+         "-schema-location", "default", "-schema-location", CRD_SCHEMAS],
+        input=manifests, capture_output=True, text=True)
+    lines = [l for l in result.stdout.splitlines() if l.strip()]
+    summary = lines[-1] if lines else ""
+    print(f"  {label}: {summary.replace('Summary: ', '')}")
+    if result.returncode != 0:
+        for l in lines[:-1]:
+            print(f"    {l}")
+        failures.append(f"{label}: {summary or result.stderr.strip()}")
+    return docs or []
+
+
+def helm_template(name, namespace, chart_args, helm):
+    with tempfile.TemporaryDirectory() as tmp:
+        args = ["helm", "template", helm.get("releaseName", name), *chart_args,
+                "--namespace", namespace, "--kube-version", KUBE_VERSION,
+                "--include-crds"]
+        if helm.get("values"):
+            values = pathlib.Path(tmp, "values.yaml")
+            values.write_text(helm["values"])
+            args += ["-f", str(values)]
+        if helm.get("valuesObject"):
+            values = pathlib.Path(tmp, "values-object.yaml")
+            values.write_text(yaml.safe_dump(helm["valuesObject"]))
+            args += ["-f", str(values)]
+        for vf in helm.get("valueFiles", []):
+            # $values/<path> points at this repo through a source with ref: values
+            args += ["-f", str(ROOT / vf.split("/", 1)[1]) if vf.startswith("$") else vf]
+        for p in helm.get("parameters", []):
+            flag = "--set-string" if p.get("forceString") else "--set"
+            args += [flag, f"{p['name']}={p['value']}"]
+        return run(args)
+
+
+def directory(path, spec):
+    include = spec.get("include", "*.yaml")
+    exclude = spec.get("exclude", "")
+    files = sorted(path.rglob("*") if spec.get("recurse") else path.glob("*"))
+    docs = []
+    for f in files:
+        if f.suffix not in (".yaml", ".yml", ".json") or not f.is_file():
+            continue
+        name = f.name
+        if not any(fnmatch.fnmatch(name, g) for g in include.strip("{}").split(",")):
+            continue
+        if exclude and any(fnmatch.fnmatch(name, g) for g in exclude.strip("{}").split(",")):
+            continue
+        docs.append(f.read_text())
+    return "\n---\n".join(docs)
+
+
+def kustomize(path, spec):
+    if not spec:
+        return run(["kustomize", "build", str(path)])
+    # The Application's kustomize options on top, as Argo CD does.
+    with tempfile.TemporaryDirectory() as tmp:
+        k = {"apiVersion": "kustomize.config.k8s.io/v1beta1", "kind": "Kustomization",
+             "resources": [os.path.relpath(path, tmp)]}
+        for key in ("patches", "images", "namespace", "commonLabels", "namePrefix", "nameSuffix"):
+            if key in spec:
+                k[key] = spec[key]
+        pathlib.Path(tmp, "kustomization.yaml").write_text(yaml.safe_dump(k))
+        return run(["kustomize", "build", "--load-restrictor", "LoadRestrictionsNone", tmp])
+
+
+def render_path(app, src, base):
+    path = base / src.get("path", ".")
+    namespace = app["spec"]["destination"].get("namespace", "default")
+    if (path / "Chart.yaml").exists():
+        if (path / "Chart.lock").exists():
+            run(["helm", "dependency", "build", str(path)])
+        return helm_template(app["metadata"]["name"], namespace, [str(path)], src.get("helm", {}))
+    if (path / "kustomization.yaml").exists() or src.get("kustomize"):
+        return kustomize(path, src.get("kustomize"))
+    return directory(path, src.get("directory", {}))
+
+
+def render(app, src, clones):
+    name = app["metadata"]["name"]
+    namespace = app["spec"]["destination"].get("namespace", "default")
+    repo = src["repoURL"]
+    if src.get("chart"):
+        if repo.startswith("http"):
+            chart_args = [src["chart"], "--repo", repo]
+        else:
+            chart_args = [f"oci://{repo.rstrip('/')}/{src['chart']}"]
+        chart_args += ["--version", src["targetRevision"]]
+        return helm_template(name, namespace, chart_args, src.get("helm", {}))
+    if src.get("ref") and "path" not in src:
+        return None  # only provides value files
+    if repo in OWN_REPOS:
+        return render_path(app, src, ROOT)
+    key = (repo, src.get("targetRevision", "HEAD"))
+    if key not in clones:
+        tmp = tempfile.mkdtemp()
+        run(["git", "clone", "--quiet", "--depth", "1", "--branch", key[1], repo, tmp])
+        clones[key] = pathlib.Path(tmp)
+    return render_path(app, src, clones[key])
+
+
+def applications(docs):
+    return [d for d in docs if isinstance(d, dict) and d.get("kind") == "Application"
+            and d.get("apiVersion", "").startswith("argoproj.io/")]
+
+
+def main():
+    print("Duplicate keys")
+    check_duplicate_keys()
+
+    print("Applications")
+    apps = applications(validate("apps (chart)", run(
+        ["helm", "template", "apps", str(ROOT / "apps"), "--kube-version", KUBE_VERSION])))
+    for path in yaml_files():
+        if path.parts[len(ROOT.parts)] == "apps":
+            continue
+        try:
+            apps += applications([d for d in yaml.safe_load_all(path.read_text()) if d])
+        except yaml.YAMLError:
+            pass  # reported by the duplicate key check or by the app that uses it
+
+    clones = {}
+    seen = set()
+    for app in sorted(apps, key=lambda a: a["metadata"]["name"]):
+        name = app["metadata"]["name"]
+        if name in seen:
+            continue
+        seen.add(name)
+        spec = app["spec"]
+        sources = spec.get("sources") or [spec["source"]]
+        for i, src in enumerate(sources):
+            label = name if len(sources) == 1 else f"{name}[{i}]"
+            try:
+                manifests = render(app, src, clones)
+            except subprocess.CalledProcessError as e:
+                print(f"  {label}: render failed")
+                lines = []
+                for line in (e.stderr or e.stdout).strip().splitlines():
+                    if not lines or line != lines[-1]:  # Helm repeats schema errors
+                        lines.append(line)
+                for line in lines[:20]:
+                    print(f"    {line}")
+                failures.append(f"{label}: render failed")
+                continue
+            if manifests is not None:
+                validate(label, manifests)
+
+    if failures:
+        print(f"\n{len(failures)} problem(s):")
+        for f in failures:
+            print(f"- {f}")
+        sys.exit(1)
+    print("\nAll apps render and validate.")
+
+
+if __name__ == "__main__":
+    main()