⎈ fix: Release Helm Charts When GitHub Refuses Their Tags (#18511)

This commit is contained in:
Danny Avila 2026-11-19 10:11:37 -05:00 • committed by GitHub
parent a9034aa6a2
commit e6cc98bf9d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 395 additions and 6 deletions

View file

@ -10,6 +10,13 @@ TAG_PREFIX="${TAG_PREFIX:-chart-}"
GITHUB_SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}"
DISPATCH_WORKFLOW="${DISPATCH_WORKFLOW:-}"
RELEASE_EXISTING_TAG="${RELEASE_EXISTING_TAG:-}"
CHART_REGISTRY="${CHART_REGISTRY:-ghcr.io}"
CHART_REPOSITORY="${CHART_REPOSITORY:-}"
# GitHub refuses a ref created by a token without the `workflows` permission when the commit holds
# workflow file changes; in practice, a workflow file version found at no branch or tag tip.
# GITHUB_TOKEN can never hold that permission, so a chart version bumped in the middle of a
# promotion that also changes workflows cannot be tagged from here.
WORKFLOW_REFUSAL='to create or update workflow'
SEMVER_REGEX='^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)(-[0-9A-Za-z-]+([.][0-9A-Za-z-]+)*)?([+][0-9A-Za-z-]+([.][0-9A-Za-z-]+)*)?$'
fail() {
@ -33,6 +40,7 @@ git_with_auth() {
dispatch_release() {
tag="$1"
release_commit="${2:-}"
if [ -z "$DISPATCH_WORKFLOW" ]; then
return
@ -58,10 +66,98 @@ dispatch_release() {
command -v gh >/dev/null ||
fail "GitHub CLI is required to dispatch ${DISPATCH_WORKFLOW}"
release_inputs=(-f "chart_tag=${tag}")
if [ -n "$release_commit" ]; then
release_inputs+=(-f "chart_sha=${release_commit}")
fi
GH_TOKEN="$token" gh workflow run "$DISPATCH_WORKFLOW" \
--repo "$GITHUB_REPOSITORY" \
--ref "$DEFAULT_BRANCH" \
-f "chart_tag=${tag}"
"${release_inputs[@]}"
}
chart_repository() {
if [ -n "$CHART_REPOSITORY" ]; then
printf '%s' "$CHART_REPOSITORY"
return
fi
if [[ ! "${GITHUB_REPOSITORY:-}" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
fail "GITHUB_REPOSITORY is required to look up published charts"
fi
# Must match the repository helmcharts.yml publishes to; OCI names are lowercase.
owner="${GITHUB_REPOSITORY%%/*}"
printf '%s/librechat-chart/librechat' "${owner,,}"
}
# Succeeds when the registry holds the chart version and returns 1 when it does not. Anything
# else is an outage, and failing keeps it from being read as either answer.
chart_published() {
version="$1"
repository="$(chart_repository)" || exit 1
token_url="https://${CHART_REGISTRY}/token?service=${CHART_REGISTRY}&scope=repository:${repository}:pull"
credentials=()
if [ -n "${GITHUB_TOKEN:-}" ]; then
credentials=(-u "x-access-token:${GITHUB_TOKEN}")
fi
registry_token="$(curl -fsS --max-time 30 "${credentials[@]}" "$token_url" | jq -er '.token')" ||
fail "Unable to get a pull token for ${CHART_REGISTRY}/${repository}"
# Helm stores SemVer build metadata with '_' because OCI tags cannot hold '+'.
status="$(curl -sS --max-time 30 -o /dev/null -w '%{http_code}' \
-H "Authorization: Bearer ${registry_token}" \
-H 'Accept: application/vnd.oci.image.manifest.v1+json' \
"https://${CHART_REGISTRY}/v2/${repository}/manifests/${version//+/_}")" ||
fail "Unable to reach ${CHART_REGISTRY} for ${repository}:${version}"
case "$status" in
200) return 0 ;;
404) return 1 ;;
*) fail "Unexpected status ${status} from ${CHART_REGISTRY} for ${repository}:${version}" ;;
esac
}
# The chart is the deliverable, so a version GitHub will not let this token tag is still released,
# once, from its commit. The tag is left for a maintainer whose credential can create it.
release_untagged() {
tag="$1"
commit="$2"
version="${tag#${TAG_PREFIX}}"
printf '::warning::GitHub refused %s at %s: this token cannot create a ref holding workflow file changes. A maintainer can run: git push origin %s:refs/tags/%s\n' \
"$tag" "$(git rev-parse --short "$commit")" "$commit" "$tag"
printf '%s\t%s\n' "$tag" "$commit" >>"$untagged_file"
if [ -z "$DISPATCH_WORKFLOW" ]; then
return
fi
if chart_published "$version"; then
printf 'Chart %s is already published; only %s is missing.\n' "$version" "$tag"
return
fi
printf 'Dispatching release workflow for %s from %s.\n' "$version" "$commit"
dispatch_release "$tag" "$commit"
}
summarize_untagged() {
if [ ! -s "$untagged_file" ] || [ -z "${GITHUB_STEP_SUMMARY:-}" ]; then
return
fi
{
printf '### Chart tags a maintainer has to push\n\n'
printf 'GitHub refuses these tags from the workflow token because the commits hold workflow file changes. The charts are released from the commits either way.\n\n'
printf '```bash\n'
while IFS="$(printf '\t')" read -r tag commit; do
printf 'git push origin %s:refs/tags/%s\n' "$commit" "$tag"
done <"$untagged_file"
printf '```\n'
} >>"$GITHUB_STEP_SUMMARY"
}
version_less_than() {
@ -147,8 +243,10 @@ history_file="$(mktemp)"
versions_file="$(mktemp)"
seen_file="$(mktemp)"
missing_file="$(mktemp)"
untagged_file="$(mktemp)"
push_log="$(mktemp)"
cleanup() {
rm -f "$history_file" "$versions_file" "$seen_file" "$missing_file"
rm -f "$history_file" "$versions_file" "$seen_file" "$missing_file" "$untagged_file" "$push_log"
}
trap cleanup EXIT
@ -219,7 +317,11 @@ while IFS="$(printf '\t')" read -r tag commit; do
git tag "$tag" "$commit"
if git_with_auth push origin "refs/tags/${tag}"; then
pushed=true
git_with_auth push origin "refs/tags/${tag}" 2>"$push_log" || pushed=false
cat "$push_log" >&2
if [ "$pushed" = true ]; then
printf 'Created %s at %s.\n' "$tag" "$short_commit"
dispatch_release "$tag"
continue
@ -231,7 +333,15 @@ while IFS="$(printf '\t')" read -r tag commit; do
continue
fi
git tag -d "$tag" >/dev/null
if grep -Fq "$WORKFLOW_REFUSAL" "$push_log"; then
release_untagged "$tag" "$commit"
continue
fi
fail "Failed to push ${tag}"
done <"$missing_file"
summarize_untagged
dispatch_existing_tag "$RELEASE_EXISTING_TAG"

View file

@ -0,0 +1,200 @@
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import {
appendFileSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { test } from 'node:test';
import { fileURLToPath } from 'node:url';
const script = join(dirname(fileURLToPath(import.meta.url)), 'sync-helm-chart-tags.sh');
const CHART_PATH = 'helm/librechat/Chart.yaml';
const WORKFLOW_REFUSAL =
'refusing to allow a GitHub App to create or update workflow `.github/workflows/ci.yml` without `workflows` permission';
/**
* A real remote whose `update` hook refuses the listed tags with `refusal`, which is how GitHub
* answers a workflow token that tags a commit holding workflow file changes. The registry and the
* GitHub CLI are the only fakes.
*/
function setup(
t,
{ refused = [], refusal = WORKFLOW_REFUSAL, published = [], registryStatus = '' },
) {
const scratch = mkdtempSync(join(tmpdir(), 'sync-helm-chart-tags-'));
t.after(() => rmSync(scratch, { recursive: true, force: true }));
const origin = join(scratch, 'origin.git');
const work = join(scratch, 'work');
const bin = join(scratch, 'bin');
const calls = join(scratch, 'calls');
const registry = join(scratch, 'registry');
const summary = join(scratch, 'summary');
const env = {
...process.env,
PATH: `${bin}:${process.env.PATH}`,
GIT_CONFIG_GLOBAL: '/dev/null',
GIT_CONFIG_NOSYSTEM: '1',
GIT_AUTHOR_NAME: 'test',
GIT_AUTHOR_EMAIL: 'test@example.com',
GIT_COMMITTER_NAME: 'test',
GIT_COMMITTER_EMAIL: 'test@example.com',
CALLS_PATH: calls,
REGISTRY_PATH: registry,
REGISTRY_STATUS: registryStatus,
GITHUB_REPOSITORY: 'LibreChat-AI/LibreChat',
GITHUB_STEP_SUMMARY: summary,
GH_TOKEN: 'test',
BACKFILL_FROM_VERSION: '1.0.0',
DISPATCH_WORKFLOW: 'helmcharts.yml',
PUSH_TAGS: 'true',
};
delete env.GITHUB_TOKEN;
const git = (cwd, ...args) => {
const result = spawnSync('git', args, { cwd, env, encoding: 'utf8' });
assert.equal(result.status, 0, `git ${args.join(' ')}\n${result.stderr}`);
return result.stdout.trim();
};
mkdirSync(bin);
writeFileSync(calls, '');
writeFileSync(registry, published.map((version) => `${version}\n`).join(''));
writeFileSync(
join(bin, 'gh'),
`#!/usr/bin/env bash
printf '%s\\n' "$*" >> "$CALLS_PATH"
`,
{ mode: 0o700 },
);
writeFileSync(
join(bin, 'curl'),
`#!/usr/bin/env bash
url="\${!#}"
case "$url" in
*/token\\?*) printf '{"token":"pull"}' ;;
https://ghcr.io/v2/librechat-ai/librechat-chart/librechat/manifests/*)
version="\${url##*/}"
status=404
if grep -Fqx "$version" "$REGISTRY_PATH"; then status=200; fi
printf '%s' "\${REGISTRY_STATUS:-$status}"
;;
*) echo "unexpected registry call: $url" >&2; exit 1 ;;
esac
`,
{ mode: 0o700 },
);
git(scratch, 'init', '--quiet', '--bare', '-b', 'main', origin);
writeFileSync(
join(origin, 'hooks', 'update'),
`#!/usr/bin/env bash
case " ${refused.map((tag) => `refs/tags/${tag}`).join(' ')} " in
*" $1 "*) echo ${JSON.stringify(refusal)} >&2; exit 1 ;;
esac
`,
{ mode: 0o700 },
);
git(scratch, 'init', '--quiet', '-b', 'main', work);
git(work, 'remote', 'add', 'origin', origin);
mkdirSync(dirname(join(work, CHART_PATH)), { recursive: true });
const commits = {};
for (const version of ['1.0.0', '1.1.0', '1.2.0']) {
writeFileSync(join(work, CHART_PATH), `name: librechat\nversion: ${version}\n`);
git(work, 'add', CHART_PATH);
git(work, 'commit', '--quiet', '-m', version);
commits[version] = git(work, 'rev-parse', 'HEAD');
}
git(work, 'push', '--quiet', 'origin', 'main');
git(work, 'fetch', '--quiet', 'origin');
return {
commits,
summary,
publish: (version) => appendFileSync(registry, `${version}\n`),
sync: () => {
const result = spawnSync('bash', [script], { cwd: work, env, encoding: 'utf8' });
return {
status: result.status,
output: `${result.stdout}\n${result.stderr}`,
dispatched: readFileSync(calls, 'utf8').trim().split('\n').filter(Boolean),
remoteTags: git(origin, 'tag', '--list').split('\n').filter(Boolean),
};
},
};
}
const dispatch = (tag, commit) =>
`workflow run helmcharts.yml --repo LibreChat-AI/LibreChat --ref main -f chart_tag=${tag}` +
(commit ? ` -f chart_sha=${commit}` : '');
test('a tag GitHub refuses does not block later versions and its chart is released from its commit', (t) => {
const { commits, summary, sync } = setup(t, { refused: ['chart-1.1.0'] });
const { status, output, dispatched, remoteTags } = sync();
assert.equal(status, 0, output);
assert.deepEqual(remoteTags, ['chart-1.0.0', 'chart-1.2.0']);
assert.deepEqual(dispatched, [
dispatch('chart-1.0.0'),
dispatch('chart-1.1.0', commits['1.1.0']),
dispatch('chart-1.2.0'),
]);
const command = `git push origin ${commits['1.1.0']}:refs/tags/chart-1.1.0`;
assert.match(output, /::warning::GitHub refused chart-1\.1\.0/);
assert.ok(output.includes(command), output);
assert.ok(readFileSync(summary, 'utf8').includes(command));
});
test('a refused tag whose chart is already published is not released again', (t) => {
const { sync } = setup(t, { refused: ['chart-1.1.0'], published: ['1.1.0'] });
const { status, output, dispatched, remoteTags } = sync();
assert.equal(status, 0, output);
assert.deepEqual(remoteTags, ['chart-1.0.0', 'chart-1.2.0']);
assert.deepEqual(dispatched, [dispatch('chart-1.0.0'), dispatch('chart-1.2.0')]);
assert.match(output, /::warning::GitHub refused chart-1\.1\.0/);
});
test('the next sync asks for the refused tag again without releasing its chart twice', (t) => {
const { commits, publish, sync } = setup(t, { refused: ['chart-1.1.0'] });
const first = sync();
publish('1.1.0');
const second = sync();
assert.equal(second.status, 0, second.output);
assert.match(second.output, /::warning::GitHub refused chart-1\.1\.0/);
assert.deepEqual(second.dispatched, first.dispatched);
assert.deepEqual(second.dispatched, [
dispatch('chart-1.0.0'),
dispatch('chart-1.1.0', commits['1.1.0']),
dispatch('chart-1.2.0'),
]);
});
test('a push that fails for any other reason still fails the sync', (t) => {
const { sync } = setup(t, { refused: ['chart-1.1.0'], refusal: 'pre-receive hook declined' });
const { status, output, dispatched, remoteTags } = sync();
assert.equal(status, 1, output);
assert.match(output, /::error::Failed to push chart-1\.1\.0/);
assert.deepEqual(remoteTags, ['chart-1.0.0']);
assert.deepEqual(dispatched, [dispatch('chart-1.0.0')]);
});
test('a registry outage is not read as an unpublished chart', (t) => {
const { summary, sync } = setup(t, { refused: ['chart-1.1.0'], registryStatus: '503' });
const { status, output, dispatched } = sync();
assert.equal(status, 1, output);
assert.match(output, /::error::Unexpected status 503 from ghcr\.io/);
assert.deepEqual(dispatched, [dispatch('chart-1.0.0')]);
assert.equal(existsSync(summary), false);
});

View file

@ -8,9 +8,13 @@ on:
workflow_dispatch:
inputs:
chart_tag:
description: "Existing chart tag to release, for example chart-2.0.5"
description: "Chart tag to release, for example chart-2.0.5"
required: true
type: string
chart_sha:
description: "Commit on main to release from when the tag could not be created"
required: false
type: string
jobs:
release:
@ -23,6 +27,7 @@ jobs:
id: chart-version
env:
EVENT_NAME: ${{ github.event_name }}
INPUT_CHART_SHA: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.chart_sha || '' }}
INPUT_CHART_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.chart_tag || '' }}
REF_NAME: ${{ github.ref_name }}
run: |
@ -39,10 +44,23 @@ jobs:
exit 1
fi
# GitHub refuses some chart tags from the tag sync's token, which then names the commit.
CHART_REF="refs/tags/${CHART_TAG}"
CHART_SHA=""
if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -n "$INPUT_CHART_SHA" ]; then
if [[ ! "$INPUT_CHART_SHA" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::chart_sha must be a full commit SHA"
exit 1
fi
CHART_REF="$INPUT_CHART_SHA"
CHART_SHA="$INPUT_CHART_SHA"
fi
# OCI repository names must be lowercase, and the repository owner is not guaranteed to be.
{
printf 'CHART_REPOSITORY=%s/librechat-chart\n' "${GITHUB_REPOSITORY_OWNER,,}"
printf 'CHART_REF=refs/tags/%s\n' "$CHART_TAG"
printf 'CHART_REF=%s\n' "$CHART_REF"
printf 'CHART_SHA=%s\n' "$CHART_SHA"
printf 'CHART_TAG=%s\n' "$CHART_TAG"
printf 'CHART_VERSION=%s\n' "$CHART_VERSION"
} >> "$GITHUB_OUTPUT"
@ -64,6 +82,25 @@ jobs:
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
# A tag names its own commit. A bare SHA has to earn the version it is released under.
- name: Verify the commit carries this chart version on main
if: steps.chart-version.outputs.CHART_SHA != ''
env:
CHART_SHA: ${{ steps.chart-version.outputs.CHART_SHA }}
CHART_VERSION: ${{ steps.chart-version.outputs.CHART_VERSION }}
run: |
set -euo pipefail
if ! git merge-base --is-ancestor "$CHART_SHA" refs/remotes/origin/main; then
echo "::error::${CHART_SHA} is not on main"
exit 1
fi
FOUND_VERSION="$(helm show chart helm/librechat | sed -n 's/^version: //p')"
if [ "$FOUND_VERSION" != "$CHART_VERSION" ]; then
echo "::error::helm/librechat/Chart.yaml at ${CHART_SHA} is version '${FOUND_VERSION}', not ${CHART_VERSION}"
exit 1
fi
- name: Build Subchart Deps
run: |
cd helm/librechat-rag-api
@ -79,9 +116,30 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# A chart released from its commit must not be replaced when a maintainer pushes its tag
# later, or when two tag syncs dispatch the same commit. Dispatching a tag by hand still
# releases it again.
- name: Find already released charts
id: released
if: github.event_name == 'push' || steps.chart-version.outputs.CHART_SHA != ''
env:
CHART_REPOSITORY: ${{ steps.chart-version.outputs.CHART_REPOSITORY }}
CHART_VERSION: ${{ steps.chart-version.outputs.CHART_VERSION }}
run: |
set -euo pipefail
for chart in librechat librechat-rag-api; do
released=false
if helm show chart "oci://ghcr.io/${CHART_REPOSITORY}/${chart}" --version "$CHART_VERSION" >/dev/null 2>&1; then
released=true
echo "::notice::${chart} ${CHART_VERSION} is already released; leaving it untouched."
fi
printf '%s=%s\n' "$chart" "$released" >> "$GITHUB_OUTPUT"
done
# Run Helm OCI Charts Releaser
# This is for the librechat chart
- name: Release Helm OCI Charts for librechat
if: steps.released.outputs.librechat != 'true'
uses: appany/helm-oci-chart-releaser@v0.4.2
with:
name: librechat
@ -94,6 +152,7 @@ jobs:
# this is for the librechat-rag-api chart
- name: Release Helm OCI Charts for librechat-rag-api
if: steps.released.outputs.librechat-rag-api != 'true'
uses: appany/helm-oci-chart-releaser@v0.4.2
with:
name: librechat-rag-api

View file

@ -6,6 +6,10 @@ on:
- main
paths-ignore:
- '**.md'
pull_request:
paths:
- '.github/scripts/sync-helm-chart-tags*'
- '.github/workflows/sync-helm-chart-tags.yml'
workflow_dispatch:
inputs:
release_existing_tag:
@ -16,8 +20,9 @@ on:
permissions:
contents: read
# Tag syncs share one group so two never push the same tag; a pull request only runs the tests.
concurrency:
group: sync-helm-chart-tags
group: sync-helm-chart-tags-${{ github.event.pull_request.number || 'main' }}
cancel-in-progress: false
jobs:
@ -30,6 +35,20 @@ jobs:
- name: Skip tag sync
run: echo "Helm chart tag sync only runs on main pushes or manual dispatch."
test:
name: Test chart tag sync
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v5
with:
persist-credentials: false
sparse-checkout: .github/scripts
- name: Test tag sync against a remote that refuses tags
run: node --test .github/scripts/sync-helm-chart-tags.test.mjs
sync:
name: Sync chart tags
if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main'
@ -38,6 +57,7 @@ jobs:
permissions:
actions: write
contents: write
packages: read
env:
BASE_REF: refs/remotes/origin/main
BACKFILL_FROM_VERSION: 1.9.0