diff --git a/.github/scripts/sync-helm-chart-tags.sh b/.github/scripts/sync-helm-chart-tags.sh index f54048a1ea..ffb0be3953 100755 --- a/.github/scripts/sync-helm-chart-tags.sh +++ b/.github/scripts/sync-helm-chart-tags.sh @@ -10,6 +10,13 @@ TAG_PREFIX="${TAG_PREFIX:-chart-}" GITHUB_SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}" DISPATCH_WORKFLOW="${DISPATCH_WORKFLOW:-}" RELEASE_EXISTING_TAG="${RELEASE_EXISTING_TAG:-}" +CHART_REGISTRY="${CHART_REGISTRY:-ghcr.io}" +CHART_REPOSITORY="${CHART_REPOSITORY:-}" +# GitHub refuses a ref created by a token without the `workflows` permission when the commit holds +# workflow file changes; in practice, a workflow file version found at no branch or tag tip. +# GITHUB_TOKEN can never hold that permission, so a chart version bumped in the middle of a +# promotion that also changes workflows cannot be tagged from here. +WORKFLOW_REFUSAL='to create or update workflow' SEMVER_REGEX='^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)(-[0-9A-Za-z-]+([.][0-9A-Za-z-]+)*)?([+][0-9A-Za-z-]+([.][0-9A-Za-z-]+)*)?$' fail() { @@ -33,6 +40,7 @@ git_with_auth() { dispatch_release() { tag="$1" + release_commit="${2:-}" if [ -z "$DISPATCH_WORKFLOW" ]; then return @@ -58,10 +66,98 @@ dispatch_release() { command -v gh >/dev/null || fail "GitHub CLI is required to dispatch ${DISPATCH_WORKFLOW}" + release_inputs=(-f "chart_tag=${tag}") + if [ -n "$release_commit" ]; then + release_inputs+=(-f "chart_sha=${release_commit}") + fi + GH_TOKEN="$token" gh workflow run "$DISPATCH_WORKFLOW" \ --repo "$GITHUB_REPOSITORY" \ --ref "$DEFAULT_BRANCH" \ - -f "chart_tag=${tag}" + "${release_inputs[@]}" +} + +chart_repository() { + if [ -n "$CHART_REPOSITORY" ]; then + printf '%s' "$CHART_REPOSITORY" + return + fi + + if [[ ! "${GITHUB_REPOSITORY:-}" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + fail "GITHUB_REPOSITORY is required to look up published charts" + fi + + # Must match the repository helmcharts.yml publishes to; OCI names are lowercase. + owner="${GITHUB_REPOSITORY%%/*}" + printf '%s/librechat-chart/librechat' "${owner,,}" +} + +# Succeeds when the registry holds the chart version and returns 1 when it does not. Anything +# else is an outage, and failing keeps it from being read as either answer. +chart_published() { + version="$1" + repository="$(chart_repository)" || exit 1 + token_url="https://${CHART_REGISTRY}/token?service=${CHART_REGISTRY}&scope=repository:${repository}:pull" + credentials=() + if [ -n "${GITHUB_TOKEN:-}" ]; then + credentials=(-u "x-access-token:${GITHUB_TOKEN}") + fi + + registry_token="$(curl -fsS --max-time 30 "${credentials[@]}" "$token_url" | jq -er '.token')" || + fail "Unable to get a pull token for ${CHART_REGISTRY}/${repository}" + + # Helm stores SemVer build metadata with '_' because OCI tags cannot hold '+'. + status="$(curl -sS --max-time 30 -o /dev/null -w '%{http_code}' \ + -H "Authorization: Bearer ${registry_token}" \ + -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ + "https://${CHART_REGISTRY}/v2/${repository}/manifests/${version//+/_}")" || + fail "Unable to reach ${CHART_REGISTRY} for ${repository}:${version}" + + case "$status" in + 200) return 0 ;; + 404) return 1 ;; + *) fail "Unexpected status ${status} from ${CHART_REGISTRY} for ${repository}:${version}" ;; + esac +} + +# The chart is the deliverable, so a version GitHub will not let this token tag is still released, +# once, from its commit. The tag is left for a maintainer whose credential can create it. +release_untagged() { + tag="$1" + commit="$2" + version="${tag#${TAG_PREFIX}}" + + printf '::warning::GitHub refused %s at %s: this token cannot create a ref holding workflow file changes. A maintainer can run: git push origin %s:refs/tags/%s\n' \ + "$tag" "$(git rev-parse --short "$commit")" "$commit" "$tag" + printf '%s\t%s\n' "$tag" "$commit" >>"$untagged_file" + + if [ -z "$DISPATCH_WORKFLOW" ]; then + return + fi + + if chart_published "$version"; then + printf 'Chart %s is already published; only %s is missing.\n' "$version" "$tag" + return + fi + + printf 'Dispatching release workflow for %s from %s.\n' "$version" "$commit" + dispatch_release "$tag" "$commit" +} + +summarize_untagged() { + if [ ! -s "$untagged_file" ] || [ -z "${GITHUB_STEP_SUMMARY:-}" ]; then + return + fi + + { + printf '### Chart tags a maintainer has to push\n\n' + printf 'GitHub refuses these tags from the workflow token because the commits hold workflow file changes. The charts are released from the commits either way.\n\n' + printf '```bash\n' + while IFS="$(printf '\t')" read -r tag commit; do + printf 'git push origin %s:refs/tags/%s\n' "$commit" "$tag" + done <"$untagged_file" + printf '```\n' + } >>"$GITHUB_STEP_SUMMARY" } version_less_than() { @@ -147,8 +243,10 @@ history_file="$(mktemp)" versions_file="$(mktemp)" seen_file="$(mktemp)" missing_file="$(mktemp)" +untagged_file="$(mktemp)" +push_log="$(mktemp)" cleanup() { - rm -f "$history_file" "$versions_file" "$seen_file" "$missing_file" + rm -f "$history_file" "$versions_file" "$seen_file" "$missing_file" "$untagged_file" "$push_log" } trap cleanup EXIT @@ -219,7 +317,11 @@ while IFS="$(printf '\t')" read -r tag commit; do git tag "$tag" "$commit" - if git_with_auth push origin "refs/tags/${tag}"; then + pushed=true + git_with_auth push origin "refs/tags/${tag}" 2>"$push_log" || pushed=false + cat "$push_log" >&2 + + if [ "$pushed" = true ]; then printf 'Created %s at %s.\n' "$tag" "$short_commit" dispatch_release "$tag" continue @@ -231,7 +333,15 @@ while IFS="$(printf '\t')" read -r tag commit; do continue fi + git tag -d "$tag" >/dev/null + + if grep -Fq "$WORKFLOW_REFUSAL" "$push_log"; then + release_untagged "$tag" "$commit" + continue + fi + fail "Failed to push ${tag}" done <"$missing_file" +summarize_untagged dispatch_existing_tag "$RELEASE_EXISTING_TAG" diff --git a/.github/scripts/sync-helm-chart-tags.test.mjs b/.github/scripts/sync-helm-chart-tags.test.mjs new file mode 100644 index 0000000000..47f3fb185f --- /dev/null +++ b/.github/scripts/sync-helm-chart-tags.test.mjs @@ -0,0 +1,200 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + appendFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const script = join(dirname(fileURLToPath(import.meta.url)), 'sync-helm-chart-tags.sh'); +const CHART_PATH = 'helm/librechat/Chart.yaml'; +const WORKFLOW_REFUSAL = + 'refusing to allow a GitHub App to create or update workflow `.github/workflows/ci.yml` without `workflows` permission'; + +/** + * A real remote whose `update` hook refuses the listed tags with `refusal`, which is how GitHub + * answers a workflow token that tags a commit holding workflow file changes. The registry and the + * GitHub CLI are the only fakes. + */ +function setup( + t, + { refused = [], refusal = WORKFLOW_REFUSAL, published = [], registryStatus = '' }, +) { + const scratch = mkdtempSync(join(tmpdir(), 'sync-helm-chart-tags-')); + t.after(() => rmSync(scratch, { recursive: true, force: true })); + const origin = join(scratch, 'origin.git'); + const work = join(scratch, 'work'); + const bin = join(scratch, 'bin'); + const calls = join(scratch, 'calls'); + const registry = join(scratch, 'registry'); + const summary = join(scratch, 'summary'); + const env = { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_NOSYSTEM: '1', + GIT_AUTHOR_NAME: 'test', + GIT_AUTHOR_EMAIL: 'test@example.com', + GIT_COMMITTER_NAME: 'test', + GIT_COMMITTER_EMAIL: 'test@example.com', + CALLS_PATH: calls, + REGISTRY_PATH: registry, + REGISTRY_STATUS: registryStatus, + GITHUB_REPOSITORY: 'LibreChat-AI/LibreChat', + GITHUB_STEP_SUMMARY: summary, + GH_TOKEN: 'test', + BACKFILL_FROM_VERSION: '1.0.0', + DISPATCH_WORKFLOW: 'helmcharts.yml', + PUSH_TAGS: 'true', + }; + delete env.GITHUB_TOKEN; + + const git = (cwd, ...args) => { + const result = spawnSync('git', args, { cwd, env, encoding: 'utf8' }); + assert.equal(result.status, 0, `git ${args.join(' ')}\n${result.stderr}`); + return result.stdout.trim(); + }; + + mkdirSync(bin); + writeFileSync(calls, ''); + writeFileSync(registry, published.map((version) => `${version}\n`).join('')); + writeFileSync( + join(bin, 'gh'), + `#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$CALLS_PATH" +`, + { mode: 0o700 }, + ); + writeFileSync( + join(bin, 'curl'), + `#!/usr/bin/env bash +url="\${!#}" +case "$url" in + */token\\?*) printf '{"token":"pull"}' ;; + https://ghcr.io/v2/librechat-ai/librechat-chart/librechat/manifests/*) + version="\${url##*/}" + status=404 + if grep -Fqx "$version" "$REGISTRY_PATH"; then status=200; fi + printf '%s' "\${REGISTRY_STATUS:-$status}" + ;; + *) echo "unexpected registry call: $url" >&2; exit 1 ;; +esac +`, + { mode: 0o700 }, + ); + + git(scratch, 'init', '--quiet', '--bare', '-b', 'main', origin); + writeFileSync( + join(origin, 'hooks', 'update'), + `#!/usr/bin/env bash +case " ${refused.map((tag) => `refs/tags/${tag}`).join(' ')} " in + *" $1 "*) echo ${JSON.stringify(refusal)} >&2; exit 1 ;; +esac +`, + { mode: 0o700 }, + ); + + git(scratch, 'init', '--quiet', '-b', 'main', work); + git(work, 'remote', 'add', 'origin', origin); + mkdirSync(dirname(join(work, CHART_PATH)), { recursive: true }); + const commits = {}; + for (const version of ['1.0.0', '1.1.0', '1.2.0']) { + writeFileSync(join(work, CHART_PATH), `name: librechat\nversion: ${version}\n`); + git(work, 'add', CHART_PATH); + git(work, 'commit', '--quiet', '-m', version); + commits[version] = git(work, 'rev-parse', 'HEAD'); + } + git(work, 'push', '--quiet', 'origin', 'main'); + git(work, 'fetch', '--quiet', 'origin'); + + return { + commits, + summary, + publish: (version) => appendFileSync(registry, `${version}\n`), + sync: () => { + const result = spawnSync('bash', [script], { cwd: work, env, encoding: 'utf8' }); + return { + status: result.status, + output: `${result.stdout}\n${result.stderr}`, + dispatched: readFileSync(calls, 'utf8').trim().split('\n').filter(Boolean), + remoteTags: git(origin, 'tag', '--list').split('\n').filter(Boolean), + }; + }, + }; +} + +const dispatch = (tag, commit) => + `workflow run helmcharts.yml --repo LibreChat-AI/LibreChat --ref main -f chart_tag=${tag}` + + (commit ? ` -f chart_sha=${commit}` : ''); + +test('a tag GitHub refuses does not block later versions and its chart is released from its commit', (t) => { + const { commits, summary, sync } = setup(t, { refused: ['chart-1.1.0'] }); + const { status, output, dispatched, remoteTags } = sync(); + + assert.equal(status, 0, output); + assert.deepEqual(remoteTags, ['chart-1.0.0', 'chart-1.2.0']); + assert.deepEqual(dispatched, [ + dispatch('chart-1.0.0'), + dispatch('chart-1.1.0', commits['1.1.0']), + dispatch('chart-1.2.0'), + ]); + + const command = `git push origin ${commits['1.1.0']}:refs/tags/chart-1.1.0`; + assert.match(output, /::warning::GitHub refused chart-1\.1\.0/); + assert.ok(output.includes(command), output); + assert.ok(readFileSync(summary, 'utf8').includes(command)); +}); + +test('a refused tag whose chart is already published is not released again', (t) => { + const { sync } = setup(t, { refused: ['chart-1.1.0'], published: ['1.1.0'] }); + const { status, output, dispatched, remoteTags } = sync(); + + assert.equal(status, 0, output); + assert.deepEqual(remoteTags, ['chart-1.0.0', 'chart-1.2.0']); + assert.deepEqual(dispatched, [dispatch('chart-1.0.0'), dispatch('chart-1.2.0')]); + assert.match(output, /::warning::GitHub refused chart-1\.1\.0/); +}); + +test('the next sync asks for the refused tag again without releasing its chart twice', (t) => { + const { commits, publish, sync } = setup(t, { refused: ['chart-1.1.0'] }); + const first = sync(); + publish('1.1.0'); + const second = sync(); + + assert.equal(second.status, 0, second.output); + assert.match(second.output, /::warning::GitHub refused chart-1\.1\.0/); + assert.deepEqual(second.dispatched, first.dispatched); + assert.deepEqual(second.dispatched, [ + dispatch('chart-1.0.0'), + dispatch('chart-1.1.0', commits['1.1.0']), + dispatch('chart-1.2.0'), + ]); +}); + +test('a push that fails for any other reason still fails the sync', (t) => { + const { sync } = setup(t, { refused: ['chart-1.1.0'], refusal: 'pre-receive hook declined' }); + const { status, output, dispatched, remoteTags } = sync(); + + assert.equal(status, 1, output); + assert.match(output, /::error::Failed to push chart-1\.1\.0/); + assert.deepEqual(remoteTags, ['chart-1.0.0']); + assert.deepEqual(dispatched, [dispatch('chart-1.0.0')]); +}); + +test('a registry outage is not read as an unpublished chart', (t) => { + const { summary, sync } = setup(t, { refused: ['chart-1.1.0'], registryStatus: '503' }); + const { status, output, dispatched } = sync(); + + assert.equal(status, 1, output); + assert.match(output, /::error::Unexpected status 503 from ghcr\.io/); + assert.deepEqual(dispatched, [dispatch('chart-1.0.0')]); + assert.equal(existsSync(summary), false); +}); diff --git a/.github/workflows/helmcharts.yml b/.github/workflows/helmcharts.yml index 659c210bec..fff92f9829 100644 --- a/.github/workflows/helmcharts.yml +++ b/.github/workflows/helmcharts.yml @@ -8,9 +8,13 @@ on: workflow_dispatch: inputs: chart_tag: - description: "Existing chart tag to release, for example chart-2.0.5" + description: "Chart tag to release, for example chart-2.0.5" required: true type: string + chart_sha: + description: "Commit on main to release from when the tag could not be created" + required: false + type: string jobs: release: @@ -23,6 +27,7 @@ jobs: id: chart-version env: EVENT_NAME: ${{ github.event_name }} + INPUT_CHART_SHA: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.chart_sha || '' }} INPUT_CHART_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.chart_tag || '' }} REF_NAME: ${{ github.ref_name }} run: | @@ -39,10 +44,23 @@ jobs: exit 1 fi + # GitHub refuses some chart tags from the tag sync's token, which then names the commit. + CHART_REF="refs/tags/${CHART_TAG}" + CHART_SHA="" + if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -n "$INPUT_CHART_SHA" ]; then + if [[ ! "$INPUT_CHART_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::chart_sha must be a full commit SHA" + exit 1 + fi + CHART_REF="$INPUT_CHART_SHA" + CHART_SHA="$INPUT_CHART_SHA" + fi + # OCI repository names must be lowercase, and the repository owner is not guaranteed to be. { printf 'CHART_REPOSITORY=%s/librechat-chart\n' "${GITHUB_REPOSITORY_OWNER,,}" - printf 'CHART_REF=refs/tags/%s\n' "$CHART_TAG" + printf 'CHART_REF=%s\n' "$CHART_REF" + printf 'CHART_SHA=%s\n' "$CHART_SHA" printf 'CHART_TAG=%s\n' "$CHART_TAG" printf 'CHART_VERSION=%s\n' "$CHART_VERSION" } >> "$GITHUB_OUTPUT" @@ -64,6 +82,25 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + # A tag names its own commit. A bare SHA has to earn the version it is released under. + - name: Verify the commit carries this chart version on main + if: steps.chart-version.outputs.CHART_SHA != '' + env: + CHART_SHA: ${{ steps.chart-version.outputs.CHART_SHA }} + CHART_VERSION: ${{ steps.chart-version.outputs.CHART_VERSION }} + run: | + set -euo pipefail + if ! git merge-base --is-ancestor "$CHART_SHA" refs/remotes/origin/main; then + echo "::error::${CHART_SHA} is not on main" + exit 1 + fi + + FOUND_VERSION="$(helm show chart helm/librechat | sed -n 's/^version: //p')" + if [ "$FOUND_VERSION" != "$CHART_VERSION" ]; then + echo "::error::helm/librechat/Chart.yaml at ${CHART_SHA} is version '${FOUND_VERSION}', not ${CHART_VERSION}" + exit 1 + fi + - name: Build Subchart Deps run: | cd helm/librechat-rag-api @@ -79,9 +116,30 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # A chart released from its commit must not be replaced when a maintainer pushes its tag + # later, or when two tag syncs dispatch the same commit. Dispatching a tag by hand still + # releases it again. + - name: Find already released charts + id: released + if: github.event_name == 'push' || steps.chart-version.outputs.CHART_SHA != '' + env: + CHART_REPOSITORY: ${{ steps.chart-version.outputs.CHART_REPOSITORY }} + CHART_VERSION: ${{ steps.chart-version.outputs.CHART_VERSION }} + run: | + set -euo pipefail + for chart in librechat librechat-rag-api; do + released=false + if helm show chart "oci://ghcr.io/${CHART_REPOSITORY}/${chart}" --version "$CHART_VERSION" >/dev/null 2>&1; then + released=true + echo "::notice::${chart} ${CHART_VERSION} is already released; leaving it untouched." + fi + printf '%s=%s\n' "$chart" "$released" >> "$GITHUB_OUTPUT" + done + # Run Helm OCI Charts Releaser # This is for the librechat chart - name: Release Helm OCI Charts for librechat + if: steps.released.outputs.librechat != 'true' uses: appany/helm-oci-chart-releaser@v0.4.2 with: name: librechat @@ -94,6 +152,7 @@ jobs: # this is for the librechat-rag-api chart - name: Release Helm OCI Charts for librechat-rag-api + if: steps.released.outputs.librechat-rag-api != 'true' uses: appany/helm-oci-chart-releaser@v0.4.2 with: name: librechat-rag-api diff --git a/.github/workflows/sync-helm-chart-tags.yml b/.github/workflows/sync-helm-chart-tags.yml index 13add36fcf..93b75ddfcc 100644 --- a/.github/workflows/sync-helm-chart-tags.yml +++ b/.github/workflows/sync-helm-chart-tags.yml @@ -6,6 +6,10 @@ on: - main paths-ignore: - '**.md' + pull_request: + paths: + - '.github/scripts/sync-helm-chart-tags*' + - '.github/workflows/sync-helm-chart-tags.yml' workflow_dispatch: inputs: release_existing_tag: @@ -16,8 +20,9 @@ on: permissions: contents: read +# Tag syncs share one group so two never push the same tag; a pull request only runs the tests. concurrency: - group: sync-helm-chart-tags + group: sync-helm-chart-tags-${{ github.event.pull_request.number || 'main' }} cancel-in-progress: false jobs: @@ -30,6 +35,20 @@ jobs: - name: Skip tag sync run: echo "Helm chart tag sync only runs on main pushes or manual dispatch." + test: + name: Test chart tag sync + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + sparse-checkout: .github/scripts + + - name: Test tag sync against a remote that refuses tags + run: node --test .github/scripts/sync-helm-chart-tags.test.mjs + sync: name: Sync chart tags if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' @@ -38,6 +57,7 @@ jobs: permissions: actions: write contents: write + packages: read env: BASE_REF: refs/remotes/origin/main BACKFILL_FROM_VERSION: 1.9.0