diff --git a/.github/ci-deps/packages-ubuntu-24.04-full.txt b/.github/ci-deps/packages-ubuntu-24.04-full.txt index 29344bd4e5..9e4cbc106c 100644 --- a/.github/ci-deps/packages-ubuntu-24.04-full.txt +++ b/.github/ci-deps/packages-ubuntu-24.04-full.txt @@ -92,6 +92,7 @@ python3-docutils python3-impacket python3-ldb python3-psutil +python3-yaml shellcheck uuid-dev valgrind diff --git a/.github/scripts/check-workflow-health.py b/.github/scripts/check-workflow-health.py new file mode 100755 index 0000000000..8151ed19f9 --- /dev/null +++ b/.github/scripts/check-workflow-health.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# Detect workflows that GitHub is failing to load, and track them in a +# single GitHub issue. +# +# When GitHub cannot load a workflow file it does not report a normal +# failure: the run ends within 0s with zero jobs, no logs, no annotations +# and no check runs. Among a few hundred other checks that is effectively +# invisible - os-check.yml sat broken on master for ten days in July 2026 +# before anyone noticed. +# +# .github/scripts/check-workflows.py guards the one cause we know about +# (the 21000 character per-run-step cap) before a change merges. This +# script is the net underneath it: it looks for the *symptom* rather than +# any particular cause, so a workflow that stops loading for a reason +# nobody anticipated still gets caught. +# +# Two signals, both cheap: +# +# 1. An active workflow whose registered name equals its path. GitHub +# stores a workflow's `name:` field once it has parsed the file, and +# resets it to the bare path when it cannot. This catches regressions +# as well as files that never parsed: os-check.yml was registered as +# "Ubuntu-Macos-Windows Tests" until it broke, then reverted to +# ".github/workflows/os-check.yml". +# +# 2. A completed run that failed with zero jobs. Prefiltered on +# created_at == updated_at (a load failure takes no measurable time) +# so only a handful of runs need the extra jobs lookup. +# +# Findings are reported into one issue, reused across runs: the body is +# rewritten each time, a comment is added only when the set of affected +# workflows actually changes, and the issue is closed automatically once +# everything loads again. That keeps a persistent problem from generating +# daily notification noise while still making a new one loud. +# +# Requires the gh CLI, authenticated (GH_TOKEN / GITHUB_TOKEN). +# +# Exit status: +# 0 every workflow loads +# 1 at least one workflow is failing to load +# 2 the check could not be carried out (gh call failed: no token, a +# revoked scope, an API outage). Distinct from 1 on purpose - "I +# found a problem" and "I could not look" need different responses. + +import argparse +import json +import subprocess +import sys +import time + +MARKER_PREFIX = "" + +ISSUE_TITLE = "CI: one or more workflows are failing to load" + +# The tracking issue is found by this label through the REST issues +# endpoint, never by searching the title and never via `gh issue list`. +# Title search is unusable outright: it runs off an asynchronous index +# and, worse, ignores --state, so it hands back closed issues and the +# monitor re-closes them forever. `gh issue list` uses GraphQL and can +# read a replica that has not caught up. +# +# The REST endpoint is not instantaneous either - a freshly created issue +# took ~2.4s to appear when measured against a live repository - so +# find_open_issue() re-checks rather than trusting one empty answer. +ISSUE_LABEL = "workflow-health" + + +def gh_api(path: str) -> object: + """GET a REST endpoint via the gh CLI and return parsed JSON.""" + out = subprocess.run(["gh", "api", "-H", "Accept: application/vnd.github+json", + path], + capture_output=True, text=True) + if out.returncode != 0: + raise RuntimeError(f"gh api {path} failed: {out.stderr.strip()}") + return json.loads(out.stdout) + + +def gh_json(args: list[str]) -> object: + out = subprocess.run(["gh"] + args, capture_output=True, text=True) + if out.returncode != 0: + raise RuntimeError(f"gh {' '.join(args)} failed: {out.stderr.strip()}") + return json.loads(out.stdout) if out.stdout.strip() else None + + +def unloadable_workflows(repo: str) -> list[dict]: + """Signal 1: active workflows whose name is just their path.""" + bad = [] + page = 1 + while True: + data = gh_api(f"repos/{repo}/actions/workflows" + f"?per_page=100&page={page}") + items = data.get("workflows", []) if isinstance(data, dict) else [] + for wf in items: + if wf.get("state") != "active": + continue + if wf.get("name") == wf.get("path"): + bad.append({"path": wf["path"], + "why": "registered name is the bare file path, " + "so GitHub has not parsed this file"}) + if len(items) < 100: + break + page += 1 + return bad + + +def zero_job_failures(repo: str, scan: int) -> list[dict]: + """Signal 2: recent completed runs that failed with no jobs at all.""" + bad = {} + seen = 0 + page = 1 + # Paginate rather than clamping to one page: a caller asking for more + # runs than fit in a single response should get them, not a quietly + # truncated scan that looks like full coverage. + while seen < scan: + data = gh_api(f"repos/{repo}/actions/runs" + f"?status=completed&per_page=100&page={page}") + runs = data.get("workflow_runs", []) if isinstance(data, dict) else [] + if not runs: + break + for run in runs[:scan - seen]: + if run.get("conclusion") != "failure": + continue + # A load failure never starts: it is created and completed in + # the same instant. Anything that actually ran is not this. + if run.get("created_at") != run.get("updated_at"): + continue + jobs = gh_api(f"repos/{repo}/actions/runs/{run['id']}/jobs") + if not isinstance(jobs, dict) or jobs.get("total_count", 1) != 0: + continue + path = run.get("path", "?") + bad.setdefault(path, { + "path": path, + "why": f"run {run['id']} on {run.get('head_branch', '?')} " + f"completed as a failure with zero jobs", + }) + seen += len(runs) + if len(runs) < 100: + break + page += 1 + return list(bad.values()) + + +def build_body(findings: list[dict], repo: str) -> str: + paths = sorted({f["path"] for f in findings}) + marker = f"{MARKER_PREFIX} {','.join(paths)} {MARKER_SUFFIX}" + lines = [ + marker, + "", + "One or more workflow files are not being loaded by GitHub " + "Actions. A workflow in this state does **not** fail loudly: its " + "runs complete within 0s with zero jobs, no logs, no annotations " + "and no check runs, so it looks like unrelated flake among the " + "other checks while the coverage it provides is silently gone.", + "", + "| Workflow | Detected by |", + "|---|---|", + ] + for f in sorted(findings, key=lambda x: x["path"]): + lines.append(f"| `{f['path']}` | {f['why']} |") + lines += [ + "", + "### What to check first", + "", + "GitHub caps a single `run:` step at 21000 characters and refuses " + "to load the whole file past that. Run " + "`.github/scripts/check-workflows.py` locally to test for it - " + "that is what broke `os-check.yml` for ten days in July 2026. " + "If the file is under the cap, the cause is something else; the " + "Actions service does not report which.", + "", + f"Opened automatically by `.github/workflows/workflow-health.yml` " + f"in {repo}. It closes itself once every workflow loads again.", + ] + return "\n".join(lines) + + +def marker_of(body: str) -> str: + for line in (body or "").splitlines(): + line = line.strip() + if line.startswith(MARKER_PREFIX): + return line + return "" + + +def ensure_label(repo: str) -> None: + """Create the tracking label if the repository does not have it.""" + subprocess.run(["gh", "label", "create", ISSUE_LABEL, "--repo", repo, + "--color", "B60205", + "--description", + "A workflow file is not being loaded by GitHub Actions"], + capture_output=True, text=True) + + +def find_open_issue(repo: str, attempts: int = 1, + delay: float = 2.0) -> dict | None: + """The open tracking issue, or None. + + Looked up by label through the REST issues endpoint - see ISSUE_LABEL + for why neither search nor `gh issue list` is usable here. The state + is re-checked on the result so a closed issue can never be picked up + and re-closed on every subsequent clean run. + + The title must match as well as the label. The label alone is not + proof of ownership: it is a normal repository label that anyone can + apply, and an adopted issue has its body overwritten wholesale and is + then closed, so a mislabelled one would lose its content. Requiring + the title means the worst case of someone retitling this issue is a + duplicate being opened, which is recoverable, rather than an + unrelated issue being destroyed, which is not. + + Even this endpoint is only eventually consistent: a freshly created + issue took ~2.4s to become visible when measured against a live + repository, so `attempts` re-checks before concluding nothing is + open. + """ + for attempt in range(attempts): + issues = gh_api(f"repos/{repo}/issues" + f"?state=open&labels={ISSUE_LABEL}&per_page=50") + if isinstance(issues, list): + for issue in issues: + # This endpoint returns pull requests as well. + if "pull_request" in issue: + continue + if str(issue.get("state", "")).lower() != "open": + continue + if issue.get("title") != ISSUE_TITLE: + continue + return {"number": issue["number"], + "title": issue.get("title", ""), + "body": issue.get("body") or ""} + if attempt + 1 < attempts: + time.sleep(delay) + return None + + +def manage_issue(repo: str, findings: list[dict]) -> int: + """Reconcile the tracking issue with the current findings. + + Returns the process exit status: non-zero while anything is still + failing to load, so the scheduled run itself goes red as a backstop + behind the issue. + """ + # Re-check on both paths. A stale "no open issue" opens a duplicate + # when there are findings, and silently skips closing a just-opened + # issue when there are none. A few seconds once a day is nothing + # against either. + issue = find_open_issue(repo, attempts=4) + + if not findings: + if issue: + subprocess.run(["gh", "issue", "comment", str(issue["number"]), + "--repo", repo, "--body", + "Every workflow loads again. Closing."], + check=True, capture_output=True, text=True) + subprocess.run(["gh", "issue", "close", str(issue["number"]), + "--repo", repo], + check=True, capture_output=True, text=True) + print(f"closed issue #{issue['number']}") + else: + print("nothing to report and no open issue") + return 0 + + body = build_body(findings, repo) + + if issue is None: + ensure_label(repo) + out = subprocess.run(["gh", "issue", "create", "--repo", repo, + "--title", ISSUE_TITLE, "--body", body, + "--label", ISSUE_LABEL], + capture_output=True, text=True) + if out.returncode != 0: + print(f"failed to open issue: {out.stderr.strip()}", + file=sys.stderr) + return 1 + print(f"opened issue: {out.stdout.strip()}") + return 1 + + changed = marker_of(issue.get("body", "")) != marker_of(body) + subprocess.run(["gh", "issue", "edit", str(issue["number"]), + "--repo", repo, "--body", body], + check=True, capture_output=True, text=True) + if changed: + # Only notify when the affected set actually moved - a standing + # problem should not generate a comment on every scheduled run. + subprocess.run(["gh", "issue", "comment", str(issue["number"]), + "--repo", repo, "--body", + "The set of workflows failing to load has changed; " + "the issue body above lists the current state."], + check=True, capture_output=True, text=True) + print(f"updated issue #{issue['number']} (set changed)") + else: + print(f"issue #{issue['number']} already tracks this; no comment") + return 1 + + +def run(opts: argparse.Namespace) -> int: + """Collect findings and reconcile the issue. See main() for exits.""" + findings = unloadable_workflows(opts.repo) + seen = {f["path"] for f in findings} + for f in zero_job_failures(opts.repo, opts.scan_runs): + if f["path"] not in seen: + findings.append(f) + + if findings: + print(f"{len(findings)} workflow(s) failing to load:") + for f in sorted(findings, key=lambda x: x["path"]): + print(f" {f['path']}: {f['why']}") + else: + print("all workflows load cleanly") + + if opts.report_only: + return 1 if findings else 0 + + return manage_issue(opts.repo, findings) + + +def main() -> int: + p = argparse.ArgumentParser( + description="Detect workflows GitHub is failing to load.") + p.add_argument("--repo", required=True, metavar="OWNER/REPO") + p.add_argument("--scan-runs", type=int, default=100, + help="how many recent completed runs to inspect for " + "zero-job failures (default 100)") + p.add_argument("--report-only", action="store_true", + help="print findings and exit; do not touch issues") + opts = p.parse_args() + + try: + return run(opts) + except (RuntimeError, subprocess.CalledProcessError) as exc: + # A failed gh call means the monitor could not do its job - a + # missing token, a revoked scope, an API outage. Say so in one + # line and exit 2: a traceback here reads like a bug in this + # script, and exiting 1 would be indistinguishable from having + # actually found a broken workflow. + print(f"error: could not query {opts.repo}: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/check-workflows.py b/.github/scripts/check-workflows.py new file mode 100755 index 0000000000..304eeba366 --- /dev/null +++ b/.github/scripts/check-workflows.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Static lint for GitHub Actions workflow and composite-action files. +# +# The check that matters: GitHub caps a single `run:` step at 21000 +# characters ("Exceeded max expression length 21000"). Exceeding it does +# not fail the step - GitHub refuses to load the entire workflow file, so +# every run of it ends in failure within 0s with zero jobs, no logs and no +# annotations. That is nearly invisible among a few hundred other checks: +# os-check.yml sat broken on master for ten days in July 2026 after an +# inlined config heredoc pushed one step from 20662 to 21813 characters. +# +# Because the cap is enforced by the Actions service rather than by the +# workflow schema, no YAML validator or actionlint run catches it. Hence +# this script. +# +# Sizes are measured the way GitHub sees them: parse the YAML, then take +# the length of the resulting `run` string. Block-scalar indentation is +# already stripped by the parser, so this needs no guessing about how the +# text was folded in the source file. +# +# Checks per file: +# * the file parses as YAML at all +# * every `run:` step is under the hard cap (error) and under the soft +# warning threshold (warning, so a growing list is noticed with +# runway left rather than at the cliff) +# +# Findings are emitted as GitHub workflow commands (::error / ::warning) +# so they surface as annotations on the run, and as plain text so the log +# is readable when run locally. + +import argparse +import pathlib +import sys + +import yaml + +# GitHub's hard limit on a single run: expression. +HARD_LIMIT = 21000 + +# Report anything this large as a warning: enough runway to move the +# offending content out of the workflow before it becomes a failure. +SOFT_LIMIT = 18000 + + +def iter_run_steps(doc: object) -> list[tuple[str, str]]: + """Yield (location, script) for every `run:` step in a parsed file. + + Covers both workflow files (jobs..steps[]) and composite actions + (runs.steps[]). Anything that is not shaped like a step list is + skipped rather than treated as an error: this script only measures + run steps, it is not a schema validator. + """ + found = [] + + def scan_steps(steps: object, where: str) -> None: + if not isinstance(steps, list): + return + for i, step in enumerate(steps): + if not isinstance(step, dict): + continue + script = step.get("run") + if not isinstance(script, str): + continue + name = step.get("name") or f"step {i + 1}" + found.append((f"{where} / {name}", script)) + + if not isinstance(doc, dict): + return found + + jobs = doc.get("jobs") + if isinstance(jobs, dict): + for job_id, job in jobs.items(): + if isinstance(job, dict): + scan_steps(job.get("steps"), f"jobs.{job_id}") + + runs = doc.get("runs") + if isinstance(runs, dict): + scan_steps(runs.get("steps"), "runs") + + return found + + +def check_file(path: pathlib.Path) -> tuple[int, int, int]: + """Lint one file. Returns (errors, warnings, largest run: step).""" + errors = 0 + warnings = 0 + biggest = 0 + + try: + doc = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + print(f"::error file={path}::not valid YAML: {exc}") + return (1, 0, 0) + + for where, script in iter_run_steps(doc): + biggest = max(biggest, len(script)) + size = len(script) + if size >= HARD_LIMIT: + over = size - HARD_LIMIT + print(f"::error file={path}::{where}: run: step is {size} " + f"characters, {over} over GitHub's {HARD_LIMIT} limit. " + f"GitHub will refuse to load this file and every run " + f"will fail in 0s with zero jobs. Move the bulk of the " + f"step out of the workflow - see .github/configs/ for " + f"the pattern used by the parallel-make-check.py " + f"workflows.") + errors += 1 + elif size >= SOFT_LIMIT: + left = HARD_LIMIT - size + print(f"::warning file={path}::{where}: run: step is {size} " + f"characters, only {left} under GitHub's {HARD_LIMIT} " + f"limit. Move content out of the workflow now - at the " + f"limit the whole file stops loading.") + warnings += 1 + + return (errors, warnings, biggest) + + +def main() -> int: + p = argparse.ArgumentParser( + description="Lint GitHub Actions workflow files for the 21000 " + "character per-run-step limit.") + p.add_argument("paths", nargs="*", metavar="FILE", + help="files to check (default: all workflows and " + "composite actions under .github/)") + opts = p.parse_args() + + if opts.paths: + paths = [pathlib.Path(f) for f in opts.paths] + else: + root = pathlib.Path(".github") + paths = sorted(root.glob("workflows/*.yml")) + paths += sorted(root.glob("workflows/*.yaml")) + paths += sorted(root.glob("actions/*/action.yml")) + paths += sorted(root.glob("actions/*/action.yaml")) + + paths = [f for f in paths if f.is_file()] + if not paths: + print("no workflow files found", file=sys.stderr) + return 1 + + errors = 0 + warnings = 0 + biggest = 0 + for path in paths: + e, w, b = check_file(path) + errors += e + warnings += w + biggest = max(biggest, b) + + print(f"checked {len(paths)} files; largest run: step is {biggest} " + f"characters (limit {HARD_LIMIT})") + if errors: + print(f"FAILED: {errors} step(s) over the limit") + return 1 + if warnings: + print(f"{warnings} step(s) approaching the limit") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/check-source-text.yml b/.github/workflows/check-source-text.yml index 0e10d8bd7c..cf3efb95da 100644 --- a/.github/workflows/check-source-text.yml +++ b/.github/workflows/check-source-text.yml @@ -6,6 +6,9 @@ name: Check Source Text # * check-source-text.sh: trailing whitespace, hard tabs in C/H, CRLF, # BOM / non-ASCII. # * bash -n + shellcheck (warning level) on shell scripts. +# * check-workflows.py: every `run:` step against GitHub's 21000 +# character cap, past which GitHub stops loading the workflow file +# altogether and its runs fail in 0s with zero jobs. # # Scope: # * pull_request: only files changed in the PR (catches new violations @@ -37,10 +40,12 @@ jobs: with: fetch-depth: 0 + # python3-yaml backs check-workflows.py, which measures run: steps + # from the parsed YAML rather than from the raw text. - name: Install shellcheck uses: ./.github/actions/install-apt-deps with: - packages: shellcheck + packages: shellcheck python3-yaml ghcr-debs-tag: ubuntu-24.04-full - name: Collect files to check @@ -65,6 +70,13 @@ jobs: echo "sh_count=$(wc -l < changed-sh.txt)" >> "$GITHUB_OUTPUT" fi + # Always over the whole set, not just PR-changed files: the cap is a + # property of each file on its own, the check takes well under a + # second for the ~110 of them, and a workflow can be pushed over the + # line by a change to a file the PR does not otherwise touch. + - name: Lint workflow files + run: ./.github/scripts/check-workflows.py + - name: Run check-source-text (PR changed files) if: github.event_name == 'pull_request' && steps.files.outputs.count != '0' run: | diff --git a/.github/workflows/workflow-health.yml b/.github/workflows/workflow-health.yml new file mode 100644 index 0000000000..dbd42660b0 --- /dev/null +++ b/.github/workflows/workflow-health.yml @@ -0,0 +1,59 @@ +name: Workflow Health + +# Catches workflows that GitHub is failing to load. +# +# Such a workflow does not fail loudly: its runs end within 0s with zero +# jobs, no logs and no annotations, so the coverage it provided is gone +# while the PR page still shows hundreds of green checks. os-check.yml sat +# broken on master for ten days in July 2026 before anyone noticed. +# +# check-source-text.yml lints workflow files pre-merge for the one cause +# we know about (the 21000 character per-run-step cap). This job is the +# net underneath that: it looks for the symptom instead of the cause, so a +# workflow that stops loading for an unanticipated reason is still caught, +# and it files a GitHub issue rather than adding one more red check that +# would blend in with the rest. +# +# Known gap: this workflow cannot detect its own failure to load. If this +# file stops loading it goes quiet in exactly the way it exists to +# prevent, and nothing here reports that. The pre-merge lint covers the +# size cause for this file as much as any other; the remaining causes are +# uncovered, and closing that properly needs a checker outside this +# repository. If the issue this opens has been quiet for a long stretch, +# confirm the workflow is still running rather than assuming all is well. + +on: + schedule: + # Daily, shortly after the weekday os-check ccache seed at 10:00 UTC + # so a load failure there is picked up in the same cycle. + - cron: '30 11 * * *' + # Lets a maintainer confirm a fix without waiting for the next cron. + workflow_dispatch: + +concurrency: + group: workflow-health + cancel-in-progress: false + +permissions: + contents: read + # Reading run/job metadata for every workflow in the repo. + actions: read + # Opening, updating and closing the tracking issue. + issues: write + +jobs: + check: + if: github.repository_owner == 'wolfssl' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v5 + + # The script exits non-zero while any workflow is failing to load, + # which is what surfaces this run as failed. The issue it files is + # the part meant to be noticed; the red run is a backstop. + - name: Detect workflows failing to load + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + .github/scripts/check-workflow-health.py --repo "$GITHUB_REPOSITORY"