Skip to content

[patch] Retry transient main verification #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
libops-agent merged 1 commit into from
Jul 17, 2026
+205 −10
Merged

[patch] Retry transient main verification #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 72 additions & 9 deletions .github/workflows/build-push.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,10 +461,31 @@ jobs:
if [[ -z "$EXPECTED_MAIN_SHA" ]]; then
return
fi
current_main="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq '.object.sha')"

local attempt
local current_main=""
for attempt in 1 2 3 4; do
if current_main="$(
gh api \
--method GET \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"repos/${GITHUB_REPOSITORY}/git/ref/heads/main" \
--jq '.object.sha' 2>/dev/null
)" && [[ "$current_main" =~ ^[0-9a-fA-F]{40}$ ]]; then
break
fi
current_main=""
if ((attempt == 4)); then
echo "Unable to verify main after 4 GitHub API attempts; refusing publication for $EXPECTED_MAIN_SHA" >&2
return 1
fi
echo "GitHub main lookup attempt $attempt/4 failed; retrying" >&2
sleep "$((1 << (attempt - 1)))"
done
if [[ "${current_main,,}" != "${EXPECTED_MAIN_SHA,,}" ]]; then
echo "Main advanced to $current_main; refusing to publish $EXPECTED_MAIN_SHA" >&2
exit 1
echo "Main advanced to $current_main; refusing publication for $EXPECTED_MAIN_SHA" >&2
return 1
fi
}
push_image() {
Expand Down Expand Up @@ -668,10 +689,31 @@ jobs:
if [[ -z "$EXPECTED_MAIN_SHA" ]]; then
return
fi
current_main="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq '.object.sha')"

local attempt
local current_main=""
for attempt in 1 2 3 4; do
if current_main="$(
gh api \
--method GET \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"repos/${GITHUB_REPOSITORY}/git/ref/heads/main" \
--jq '.object.sha' 2>/dev/null
)" && [[ "$current_main" =~ ^[0-9a-fA-F]{40}$ ]]; then
break
fi
current_main=""
if ((attempt == 4)); then
echo "Unable to verify main after 4 GitHub API attempts; refusing publication for $EXPECTED_MAIN_SHA" >&2
return 1
fi
echo "GitHub main lookup attempt $attempt/4 failed; retrying" >&2
sleep "$((1 << (attempt - 1)))"
done
if [[ "${current_main,,}" != "${EXPECTED_MAIN_SHA,,}" ]]; then
echo "Main advanced to $current_main; refusing to publish $EXPECTED_MAIN_SHA" >&2
exit 1
echo "Main advanced to $current_main; refusing publication for $EXPECTED_MAIN_SHA" >&2
return 1
fi
}
create_manifest() {
Expand Down Expand Up @@ -824,10 +866,31 @@ jobs:
if [[ -z "$EXPECTED_MAIN_SHA" ]]; then
return
fi
current_main="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/main" --jq '.object.sha')"

local attempt
local current_main=""
for attempt in 1 2 3 4; do
if current_main="$(
gh api \
--method GET \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"repos/${GITHUB_REPOSITORY}/git/ref/heads/main" \
--jq '.object.sha' 2>/dev/null
)" && [[ "$current_main" =~ ^[0-9a-fA-F]{40}$ ]]; then
break
fi
current_main=""
if ((attempt == 4)); then
echo "Unable to verify main after 4 GitHub API attempts; refusing publication for $EXPECTED_MAIN_SHA" >&2
return 1
fi
echo "GitHub main lookup attempt $attempt/4 failed; retrying" >&2
sleep "$((1 << (attempt - 1)))"
done
if [[ "${current_main,,}" != "${EXPECTED_MAIN_SHA,,}" ]]; then
echo "Main advanced to $current_main; refusing to sign $EXPECTED_MAIN_SHA" >&2
exit 1
echo "Main advanced to $current_main; refusing publication for $EXPECTED_MAIN_SHA" >&2
return 1
fi
}
sign_and_verify() {
Expand Down
134 changes: 133 additions & 1 deletion ci/github/build_push_contract_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,21 @@ func workflowRunScript(t *testing.T, step string) string {
return strings.TrimSpace(strings.Join(lines, "\n")) + "\n"
}

func workflowShellFunction(t *testing.T, step, name, following string) string {
t.Helper()
script := workflowRunScript(t, step)
startMarker := name + "() {\n"
start := strings.Index(script, startMarker)
if start == -1 {
t.Fatalf("workflow script is missing shell function %s", name)
}
end := strings.Index(script[start:], "\n"+following+"() {")
if end == -1 {
t.Fatalf("workflow script is missing %s after shell function %s", following, name)
}
return script[start : start+end]
}

func artifactKey(repository, image, tag, aliases, runID string) string {
value := strings.Join([]string{repository, image, tag, aliases, runID}, "\n") + "\n"
return fmt.Sprintf("%x", sha256.Sum256([]byte(value)))[:24]
Expand Down Expand Up @@ -150,7 +165,7 @@ func TestExpectedMainGuardsEveryRegistryWrite(t *testing.T) {
workflow := workflowSource(t)
push := stepBlock(t, workflow, " - name: Push verified native image\n", " - name: Preserve platform digests\n")
merge := stepBlock(t, workflow, " - name: merge platform images\n", " - name: Verify final manifest parity\n")
sign := workflow[strings.Index(workflow, " - name: Sign and verify final manifests\n"):]
sign := stepBlock(t, workflow, " - name: Sign and verify final manifests\n", "\n cleanup:\n")

requireContains(t, push, "require_current_main\n docker push \"$target\"")
guardedCreate := regexp.MustCompile(`require_current_main\s+docker buildx imagetools create`)
Expand All @@ -160,6 +175,123 @@ func TestExpectedMainGuardsEveryRegistryWrite(t *testing.T) {
requireContains(t, sign, "require_current_main\n cosign sign --yes")
}

func TestExpectedMainLookupRetriesTransientResponsesAndFailsClosed(t *testing.T) {
workflow := workflowSource(t)
push := stepBlock(t, workflow, " - name: Push verified native image\n", " - name: Preserve platform digests\n")
merge := stepBlock(t, workflow, " - name: merge platform images\n", " - name: Verify final manifest parity\n")
sign := stepBlock(t, workflow, " - name: Sign and verify final manifests\n", "\n cleanup:\n")
pushGuard := workflowShellFunction(t, push, "require_current_main", "push_image")
mergeGuard := workflowShellFunction(t, merge, "require_current_main", "create_manifest")
signGuard := workflowShellFunction(t, sign, "require_current_main", "sign_and_verify")
if pushGuard != mergeGuard || pushGuard != signGuard {
t.Fatal("every publication phase must use the same main-tip guard")
}
for _, required := range []string{
"for attempt in 1 2 3 4; do",
`[[ "$current_main" =~ ^[0-9a-fA-F]{40}$ ]]`,
"if ((attempt == 4)); then",
`return 1`,
`sleep "$((1 << (attempt - 1)))"`,
} {
requireContains(t, pushGuard, required)
}

testDir := t.TempDir()
guardPath := filepath.Join(testDir, "main-guard.sh")
guardScript := "set -euo pipefail\n" + pushGuard + "\nrequire_current_main\n"
if err := os.WriteFile(guardPath, []byte(guardScript), 0700); err != nil {
t.Fatal(err)
}
fakeBin := filepath.Join(testDir, "bin")
if err := os.Mkdir(fakeBin, 0700); err != nil {
t.Fatal(err)
}
fakeGH := `#!/usr/bin/env bash
set -euo pipefail
count=0
if [[ -f "$FAKE_GH_COUNT" ]]; then count="$(< "$FAKE_GH_COUNT")"; fi
count=$((count + 1))
printf '%s\n' "$count" > "$FAKE_GH_COUNT"
case "$FAKE_GH_MODE" in
recover)
case "$count" in
1) printf '<html>transient proxy response</html>\n'; exit 1 ;;
2) printf 'not-a-sha\n' ;;
*) printf '%s\n' "$EXPECTED_MAIN_SHA" ;;
esac
;;
mismatch) printf 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n' ;;
exhausted) printf '<html>persistent proxy response</html>\n'; exit 1 ;;
*) exit 2 ;;
esac
`
fakeSleep := `#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' "$1" >> "$FAKE_SLEEP_LOG"
`
if err := os.WriteFile(filepath.Join(fakeBin, "gh"), []byte(fakeGH), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(fakeBin, "sleep"), []byte(fakeSleep), 0700); err != nil {
t.Fatal(err)
}

type result struct {
attempts int
err error
output string
sleeps string
}
run := func(mode, expected string) result {
t.Helper()
countPath := filepath.Join(testDir, "gh-count")
sleepPath := filepath.Join(testDir, "sleep-log")
for _, path := range []string{countPath, sleepPath} {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
}
cmd := exec.Command("bash", guardPath) // #nosec G204 -- fixed test-owned script.
cmd.Env = []string{
"PATH=" + fakeBin + string(os.PathListSeparator) + "/usr/bin:/bin",
"EXPECTED_MAIN_SHA=" + expected,
"GITHUB_REPOSITORY=libops/example",
"FAKE_GH_COUNT=" + countPath,
"FAKE_GH_MODE=" + mode,
"FAKE_SLEEP_LOG=" + sleepPath,
}
output, err := cmd.CombinedOutput()
value := result{err: err, output: string(output)}
if count, readErr := os.ReadFile(countPath); readErr == nil {
if _, scanErr := fmt.Sscanf(string(count), "%d", &value.attempts); scanErr != nil {
t.Fatal(scanErr)
}
} else if !os.IsNotExist(readErr) {
t.Fatal(readErr)
}
if sleeps, readErr := os.ReadFile(sleepPath); readErr == nil {
value.sleeps = string(sleeps)
} else if !os.IsNotExist(readErr) {
t.Fatal(readErr)
}
return value
}

const expected = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
if got := run("recover", expected); got.err != nil || got.attempts != 3 || got.sleeps != "1\n2\n" {
t.Fatalf("transient recovery = attempts %d, sleeps %q, error %v, output %q", got.attempts, got.sleeps, got.err, got.output)
}
if got := run("exhausted", expected); got.err == nil || got.attempts != 4 || got.sleeps != "1\n2\n4\n" || !strings.Contains(got.output, "Unable to verify main after 4 GitHub API attempts") {
t.Fatalf("exhausted lookup = attempts %d, sleeps %q, error %v, output %q", got.attempts, got.sleeps, got.err, got.output)
}
if got := run("mismatch", expected); got.err == nil || got.attempts != 1 || got.sleeps != "" || !strings.Contains(got.output, "Main advanced to bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") {
t.Fatalf("mismatch = attempts %d, sleeps %q, error %v, output %q", got.attempts, got.sleeps, got.err, got.output)
}
if got := run("recover", ""); got.err != nil || got.attempts != 0 {
t.Fatalf("unguarded compatibility mode = attempts %d, error %v, output %q", got.attempts, got.err, got.output)
}
}

func TestDigestArtifactsRepairMixedAttempts(t *testing.T) {
workflow := workflowSource(t)
buildMetadata := stepBlock(t, workflow, " - name: Resolve publication metadata\n", " - name: GHCR Login\n")
Expand Down