🔌 refactor: Extract Git Repository Adapter From Skill Sync (#15052)

* 🔌 refactor: Extract Git Repository Adapter From Skill Sync

Skill sync interleaved GitHub REST calls with orchestration that is not
GitHub-specific in any way — discovery, import limits, upsert and stale
reconciliation, status accounting. Adding a second provider meant either
threading provider branches through that orchestration or forking it.

Introduces `GitRepoAdapter` — `resolveCommit`, `fetchTreeEntries`,
`fetchFileContent` over a normalized `RepoTreeEntry` — and moves the GitHub
REST client behind it. The runner keeps its GitHub source typing; only the
transport moved.

No behavior change: every pre-existing sync test passes untouched, still
driving real GitHub responses through the mocked `fetchFn`.

* 🔧 fix: Export GitHubRepoAdapterConfig alongside GitRepoAdapter

Self-review: the exported `createAdapter` dep names a config type that
consumers could not import, leaving half its signature unnameable.
This commit is contained in:
Danny Avila 2026-08-21 01:14:11 -04:00 committed by GitHub
parent d0f9d5625e
commit 4c45d156af
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 833 additions and 283 deletions

View file

@ -0,0 +1,332 @@
import type { SkillSyncGitHubSourceConfig } from 'librechat-data-provider';
import type { RepoCommit } from './types';
import { createGitHubRepoAdapter } from './github';
function response(body: unknown, status = 200, headers: Record<string, string> = {}): Response {
const normalizedHeaders = new Map(
Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]),
);
return {
ok: status >= 200 && status < 300,
status,
headers: {
get: (key: string) => normalizedHeaders.get(key.toLowerCase()) ?? null,
},
json: async () => body,
} as unknown as Response;
}
function treeEntry(overrides: Record<string, unknown> = {}) {
return {
path: 'SKILL.md',
mode: '100644',
type: 'blob',
sha: 'blob-sha',
size: 12,
url: 'https://api.github.test/blob',
...overrides,
};
}
function createSource(
overrides: Partial<SkillSyncGitHubSourceConfig> = {},
): SkillSyncGitHubSourceConfig {
return {
id: 'librechat-skills',
owner: 'LibreChat',
repo: 'skills',
ref: 'main',
paths: ['skills'],
credentialKey: 'github-skills-prod',
...overrides,
};
}
function createAdapter(fetchFn: typeof fetch, source = createSource()) {
return createGitHubRepoAdapter({ source, token: 'github_pat_secret', fetchFn });
}
const commit: RepoCommit = { id: 'commit-sha', treeId: 'tree-sha' };
describe('createGitHubRepoAdapter', () => {
describe('resolveCommit', () => {
it('resolves the configured ref to its commit and root tree', async () => {
const fetchFn = jest.fn(async () =>
response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }),
) as unknown as typeof fetch;
await expect(createAdapter(fetchFn).resolveCommit()).resolves.toEqual({
id: 'commit-sha',
treeId: 'tree-sha',
});
expect(fetchFn).toHaveBeenCalledWith(
'https://api.github.com/repos/LibreChat/skills/commits/main',
{
headers: expect.objectContaining({
Accept: 'application/vnd.github+json',
Authorization: 'Bearer github_pat_secret',
'X-GitHub-Api-Version': '2022-11-28',
'User-Agent': 'LibreChat-Skill-Sync',
}),
},
);
});
it('encodes each ref segment without escaping its separators', async () => {
const fetchFn = jest.fn(async () =>
response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }),
) as unknown as typeof fetch;
await createAdapter(fetchFn, createSource({ ref: 'release/v1 rc' })).resolveCommit();
expect(fetchFn).toHaveBeenCalledWith(
'https://api.github.com/repos/LibreChat/skills/commits/release/v1%20rc',
expect.anything(),
);
});
});
describe('fetchTreeEntries', () => {
it('lists the whole repository from the root tree when no path is configured', async () => {
const fetchFn = jest.fn(async () =>
response({ sha: 'tree-sha', truncated: false, tree: [treeEntry()] }),
) as unknown as typeof fetch;
const entries = await createAdapter(fetchFn).fetchTreeEntries(commit, {
pathPrefix: '',
assertNotCancelled: () => undefined,
});
expect(entries).toEqual([{ path: 'SKILL.md', type: 'blob', id: 'blob-sha', size: 12 }]);
expect(fetchFn).toHaveBeenCalledTimes(1);
expect(fetchFn).toHaveBeenCalledWith(
'https://api.github.com/repos/LibreChat/skills/git/trees/tree-sha?recursive=1',
expect.anything(),
);
});
it('walks one non-recursive listing per path segment before listing recursively', async () => {
const fetchFn = jest.fn(async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes('/git/trees/tree-sha')) {
return response({
sha: 'tree-sha',
truncated: false,
tree: [treeEntry({ path: 'skills', type: 'tree', sha: 'skills-tree-sha' })],
});
}
if (url.includes('/git/trees/skills-tree-sha')) {
return response({
sha: 'skills-tree-sha',
truncated: false,
tree: [treeEntry({ path: 'shared', type: 'tree', sha: 'shared-tree-sha' })],
});
}
return response({
sha: 'shared-tree-sha',
truncated: false,
tree: [treeEntry({ path: 'research/SKILL.md', sha: 'skill-md-sha' })],
});
}) as unknown as typeof fetch;
const entries = await createAdapter(fetchFn).fetchTreeEntries(commit, {
pathPrefix: 'skills/shared',
assertNotCancelled: () => undefined,
});
expect(entries).toEqual([
{ path: 'skills/shared/research/SKILL.md', type: 'blob', id: 'skill-md-sha', size: 12 },
]);
const urls = (fetchFn as unknown as jest.Mock).mock.calls.map(([input]) => String(input));
expect(urls).toEqual([
'https://api.github.com/repos/LibreChat/skills/git/trees/tree-sha',
'https://api.github.com/repos/LibreChat/skills/git/trees/skills-tree-sha',
'https://api.github.com/repos/LibreChat/skills/git/trees/shared-tree-sha?recursive=1',
]);
});
it('reports a configured path that no tree segment matches', async () => {
const fetchFn = jest.fn(async () =>
response({ sha: 'tree-sha', truncated: false, tree: [treeEntry()] }),
) as unknown as typeof fetch;
await expect(
createAdapter(fetchFn).fetchTreeEntries(commit, {
pathPrefix: 'skills',
assertNotCancelled: () => undefined,
}),
).rejects.toMatchObject({ name: 'SkillSyncError', code: 'GITHUB_PATH_NOT_FOUND' });
});
it('refuses a truncated listing rather than syncing a partial repository', async () => {
const fetchFn = jest.fn(async () =>
response({ sha: 'tree-sha', truncated: true, tree: [treeEntry()] }),
) as unknown as typeof fetch;
await expect(
createAdapter(fetchFn).fetchTreeEntries(commit, {
pathPrefix: '',
assertNotCancelled: () => undefined,
}),
).rejects.toMatchObject({ name: 'SkillSyncError', code: 'GITHUB_TREE_TRUNCATED' });
});
it('reports submodule entries as neither files nor directories', async () => {
const fetchFn = jest.fn(async () =>
response({
sha: 'tree-sha',
truncated: false,
tree: [
treeEntry({ path: 'vendor', type: 'commit', sha: 'submodule-sha', size: undefined }),
treeEntry({ path: 'skills', type: 'tree', sha: 'dir-sha', size: undefined }),
],
}),
) as unknown as typeof fetch;
const entries = await createAdapter(fetchFn).fetchTreeEntries(commit, {
pathPrefix: '',
assertNotCancelled: () => undefined,
});
expect(entries.map((entry) => entry.type)).toEqual(['submodule', 'tree']);
});
it('checks for cancellation between round trips', async () => {
const fetchFn = jest.fn(async (input: RequestInfo | URL) =>
String(input).includes('recursive=1')
? response({ sha: 'skills-tree-sha', truncated: false, tree: [treeEntry()] })
: response({
sha: 'tree-sha',
truncated: false,
tree: [treeEntry({ path: 'skills', type: 'tree', sha: 'skills-tree-sha' })],
}),
) as unknown as typeof fetch;
const assertNotCancelled = jest.fn();
await createAdapter(fetchFn).fetchTreeEntries(commit, {
pathPrefix: 'skills',
assertNotCancelled,
});
expect(assertNotCancelled.mock.calls.length).toBeGreaterThanOrEqual(4);
});
it('stops listing as soon as cancellation is signalled', async () => {
const fetchFn = jest.fn(async () =>
response({
sha: 'tree-sha',
truncated: false,
tree: [treeEntry({ path: 'skills', type: 'tree', sha: 'skills-tree-sha' })],
}),
) as unknown as typeof fetch;
await expect(
createAdapter(fetchFn).fetchTreeEntries(commit, {
pathPrefix: 'skills',
assertNotCancelled: () => {
throw new Error('cancelled');
},
}),
).rejects.toThrow('cancelled');
expect(fetchFn).not.toHaveBeenCalled();
});
});
describe('fetchFileContent', () => {
it('decodes base64 blob content, ignoring the wrapping whitespace GitHub inserts', async () => {
const fetchFn = jest.fn(async () =>
response({
sha: 'blob-sha',
encoding: 'base64',
size: 5,
content: `${Buffer.from('hello').toString('base64')}\n`,
}),
) as unknown as typeof fetch;
const buffer = await createAdapter(fetchFn).fetchFileContent(commit, {
path: 'skills/research/SKILL.md',
type: 'blob',
id: 'skill-md-sha',
});
expect(buffer.toString('utf-8')).toBe('hello');
expect(fetchFn).toHaveBeenCalledWith(
'https://api.github.com/repos/LibreChat/skills/git/blobs/skill-md-sha',
expect.anything(),
);
});
it('refuses a blob encoding it cannot decode', async () => {
const fetchFn = jest.fn(async () =>
response({ sha: 'blob-sha', encoding: 'utf-8', size: 5, content: 'hello' }),
) as unknown as typeof fetch;
await expect(
createAdapter(fetchFn).fetchFileContent(commit, {
path: 'skills/research/SKILL.md',
type: 'blob',
id: 'skill-md-sha',
}),
).rejects.toMatchObject({ name: 'SkillSyncError', code: 'GITHUB_UNSUPPORTED_BLOB' });
});
});
describe('error classification', () => {
it.each([
{
label: 'an exhausted rate limit budget',
status: 403,
headers: { 'x-ratelimit-remaining': '0' },
code: 'GITHUB_RATE_LIMITED',
},
{
label: 'a retry-after directive',
status: 403,
headers: { 'retry-after': '60' },
code: 'GITHUB_RATE_LIMITED',
},
{
label: 'an explicit rate limit status',
status: 429,
headers: {},
code: 'GITHUB_RATE_LIMITED',
},
{ label: 'a rejected credential', status: 401, headers: {}, code: 'GITHUB_AUTH_FAILED' },
{ label: 'a forbidden repository', status: 403, headers: {}, code: 'GITHUB_AUTH_FAILED' },
{ label: 'a missing repository', status: 404, headers: {}, code: 'GITHUB_NOT_FOUND' },
{ label: 'an upstream outage', status: 500, headers: {}, code: 'GITHUB_REQUEST_FAILED' },
])('maps $label to $code', async ({ status, headers, code }) => {
const fetchFn = jest.fn(async () =>
response({ message: 'nope' }, status, headers as Record<string, string>),
) as unknown as typeof fetch;
await expect(createAdapter(fetchFn).resolveCommit()).rejects.toMatchObject({
name: 'SkillSyncError',
code,
});
});
it('treats a rate limit explained only in the body as a rate limit', async () => {
const fetchFn = jest.fn(async () =>
response({ message: 'You have exceeded a secondary rate limit' }, 403),
) as unknown as typeof fetch;
await expect(createAdapter(fetchFn).resolveCommit()).rejects.toMatchObject({
name: 'SkillSyncError',
code: 'GITHUB_RATE_LIMITED',
});
});
it('reports a transport failure without leaking the underlying error', async () => {
const fetchFn = jest.fn(async () => {
throw new Error('ECONNREFUSED 140.82.121.6:443');
}) as unknown as typeof fetch;
await expect(createAdapter(fetchFn).resolveCommit()).rejects.toMatchObject({
name: 'SkillSyncError',
code: 'GITHUB_REQUEST_FAILED',
message: expect.not.stringContaining('ECONNREFUSED'),
});
});
});
});

View file

@ -0,0 +1,245 @@
import type { SkillSyncGitHubSourceConfig } from 'librechat-data-provider';
import type {
RepoCommit,
RepoTreeEntry,
GitRepoAdapter,
AssertNotCancelled,
RepoTreeEntryType,
FetchTreeEntriesParams,
} from './types';
import { normalizeRepoPath } from '../path';
import { SkillSyncError } from '../errors';
const GITHUB_API_BASE = 'https://api.github.com';
export const GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION =
'Use a GitHub fine-grained personal access token scoped to the selected repository with read-only Contents and Metadata permissions.';
type FetchFn = typeof fetch;
type GitHubTreeEntry = {
path: string;
mode: string;
type: 'blob' | 'tree' | 'commit';
sha: string;
size?: number;
url: string;
};
type GitHubTreeResponse = {
sha: string;
tree: GitHubTreeEntry[];
truncated: boolean;
};
type GitHubBlobResponse = {
sha: string;
content: string;
encoding: string;
size: number;
};
type GitHubCommitResponse = {
sha: string;
commit: {
tree: {
sha: string;
};
};
};
export type GitHubRepoAdapterConfig = {
source: SkillSyncGitHubSourceConfig;
token: string;
fetchFn: FetchFn;
};
function buildGitHubHeaders(token: string): HeadersInit {
return {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${token}`,
'X-GitHub-Api-Version': '2022-11-28',
'User-Agent': 'LibreChat-Skill-Sync',
};
}
function buildGitHubUrl(pathname: string): string {
return `${GITHUB_API_BASE}${pathname}`;
}
function encodeGitHubPath(value: string): string {
return value.split('/').map(encodeURIComponent).join('/');
}
async function readGitHubErrorMessage(response: Response): Promise<string | undefined> {
try {
const body = (await response.json()) as { message?: unknown };
return typeof body.message === 'string' ? body.message : undefined;
} catch {
return undefined;
}
}
function isGitHubRateLimitResponse(params: {
status: number;
remaining: string | null;
retryAfter: string | null;
message?: string;
}): boolean {
if (params.status === 429 || params.remaining === '0' || params.retryAfter) {
return true;
}
const message = params.message?.toLowerCase() ?? '';
return message.includes('rate limit') || message.includes('abuse detection');
}
async function githubJson<T>(params: {
fetchFn: FetchFn;
token: string;
pathname: string;
}): Promise<T> {
let response: Response;
try {
response = await params.fetchFn(buildGitHubUrl(params.pathname), {
headers: buildGitHubHeaders(params.token),
});
} catch {
throw new SkillSyncError(
'GITHUB_REQUEST_FAILED',
'GitHub request failed before receiving a response',
);
}
if (response.ok) {
return (await response.json()) as T;
}
const remaining = response.headers.get('x-ratelimit-remaining');
const retryAfter = response.headers.get('retry-after');
const message = await readGitHubErrorMessage(response);
if (response.status === 401 || response.status === 403 || response.status === 429) {
const code = isGitHubRateLimitResponse({
status: response.status,
remaining,
retryAfter,
message,
})
? 'GITHUB_RATE_LIMITED'
: 'GITHUB_AUTH_FAILED';
throw new SkillSyncError(code, `GitHub request failed with HTTP ${response.status}`);
}
if (response.status === 404) {
throw new SkillSyncError('GITHUB_NOT_FOUND', 'GitHub repository, ref, or path was not found');
}
throw new SkillSyncError(
'GITHUB_REQUEST_FAILED',
`GitHub request failed with HTTP ${response.status}`,
);
}
/** GitHub reports submodules as `commit` entries; every other type maps across as-is. */
function toRepoTreeEntryType(type: GitHubTreeEntry['type']): RepoTreeEntryType {
return type === 'commit' ? 'submodule' : type;
}
function toRepoTreeEntry(entry: GitHubTreeEntry, path: string): RepoTreeEntry {
return {
path,
type: toRepoTreeEntryType(entry.type),
id: entry.sha,
size: entry.size,
};
}
export function createGitHubRepoAdapter(config: GitHubRepoAdapterConfig): GitRepoAdapter {
const { source, token, fetchFn } = config;
const owner = encodeURIComponent(source.owner);
const repo = encodeURIComponent(source.repo);
async function fetchTree(treeSha: string, recursive: boolean): Promise<GitHubTreeResponse> {
return githubJson<GitHubTreeResponse>({
fetchFn,
token,
pathname: `/repos/${owner}/${repo}/git/trees/${encodeURIComponent(treeSha)}${
recursive ? '?recursive=1' : ''
}`,
});
}
/**
* GitHub addresses trees by SHA rather than by path, so reaching a configured
* subdirectory means walking one non-recursive listing per path segment before
* the recursive listing that actually enumerates it.
*/
async function resolveTreeShaAtPath(
rootTreeSha: string,
normalizedPath: string,
assertNotCancelled: AssertNotCancelled,
): Promise<string> {
let treeSha = rootTreeSha;
for (const segment of normalizedPath.split('/')) {
assertNotCancelled();
const tree = await fetchTree(treeSha, false);
assertNotCancelled();
if (tree.truncated) {
throw new SkillSyncError('GITHUB_TREE_TRUNCATED', 'GitHub tree response was truncated');
}
const next = tree.tree.find((entry) => entry.type === 'tree' && entry.path === segment);
if (!next) {
throw new SkillSyncError(
'GITHUB_PATH_NOT_FOUND',
`Configured GitHub skill path "${normalizedPath}" was not found`,
);
}
treeSha = next.sha;
}
return treeSha;
}
async function resolveCommit(): Promise<RepoCommit> {
const commit = await githubJson<GitHubCommitResponse>({
fetchFn,
token,
pathname: `/repos/${owner}/${repo}/commits/${encodeGitHubPath(source.ref)}`,
});
return { id: commit.sha, treeId: commit.commit.tree.sha };
}
async function fetchTreeEntries(
commit: RepoCommit,
params: FetchTreeEntriesParams,
): Promise<RepoTreeEntry[]> {
const normalizedPath = normalizeRepoPath(params.pathPrefix);
const treeSha = normalizedPath
? await resolveTreeShaAtPath(commit.treeId, normalizedPath, params.assertNotCancelled)
: commit.treeId;
params.assertNotCancelled();
const tree = await fetchTree(treeSha, true);
params.assertNotCancelled();
if (tree.truncated) {
throw new SkillSyncError('GITHUB_TREE_TRUNCATED', 'GitHub tree response was truncated');
}
if (!normalizedPath) {
return tree.tree.map((entry) => toRepoTreeEntry(entry, entry.path));
}
return tree.tree.map((entry) =>
toRepoTreeEntry(entry, `${normalizedPath}/${normalizeRepoPath(entry.path)}`),
);
}
async function fetchFileContent(_commit: RepoCommit, entry: RepoTreeEntry): Promise<Buffer> {
const blob = await githubJson<GitHubBlobResponse>({
fetchFn,
token,
pathname: `/repos/${owner}/${repo}/git/blobs/${encodeURIComponent(entry.id)}`,
});
if (blob.encoding !== 'base64') {
throw new SkillSyncError(
'GITHUB_UNSUPPORTED_BLOB',
`Unsupported GitHub blob encoding "${blob.encoding}"`,
);
}
return Buffer.from(blob.content.replace(/\s/g, ''), 'base64');
}
return { resolveCommit, fetchTreeEntries, fetchFileContent };
}

View file

@ -0,0 +1,84 @@
/**
* `submodule` covers entries that are neither readable files nor descendable
* directories (a Git submodule, which GitHub reports as `commit`). They are
* carried through rather than dropped so path-existence checks keep seeing
* everything the repository actually contains.
*/
export type RepoTreeEntryType = 'blob' | 'tree' | 'submodule';
/**
* One entry in a repository tree, normalized across providers.
*
* `id` is the provider's identifier for the entry's content (a GitHub blob SHA,
* a GitLab blob id). It is round-tripped back into `fetchFileContent` and
* persisted in `sourceMetadata`, which lets a later run skip re-downloading a
* file whose id has not moved. Treat it as opaque outside the adapter.
*
* `size` is the blob's byte length when the provider reports it in the tree
* listing. Skill import limits are enforced against it before any content is
* downloaded, so an adapter whose tree endpoint omits size must populate it
* some other way rather than leave it undefined.
*/
export type RepoTreeEntry = {
path: string;
type: RepoTreeEntryType;
id: string;
size?: number;
};
/** The commit a single sync run is pinned to. */
export type RepoCommit = {
/** Persisted as the run's `commitSha`; opaque outside the adapter. */
id: string;
/**
* Root tree identifier as of this commit where `fetchTreeEntries` starts
* walking. Distinct from `id` on providers whose commit and tree objects are
* addressed separately (GitHub); equal to `id` where they are not.
*/
treeId: string;
};
/**
* Throws when the run has been superseded or shut down. Adapters call it
* between network round trips so a long listing or download loop stops
* promptly instead of running to completion against a cancelled run.
*/
export type AssertNotCancelled = () => void;
export type FetchTreeEntriesParams = {
/** Repository-root-relative directory to list, or `''` for the whole repository. */
pathPrefix: string;
assertNotCancelled: AssertNotCancelled;
};
/**
* The provider-specific surface a skill sync source needs, and nothing more.
* Everything else in a run skill discovery, import limits, database upsert
* and reconciliation, status accounting is provider-agnostic and depends only
* on this interface.
*
* An adapter is bound to one configured source (its repository coordinates,
* ref, and credentials) when constructed, so no call takes them again.
*/
export interface GitRepoAdapter {
/**
* Resolves the source's configured ref to the commit that pins this run, so
* every entry listed and file fetched within it stays consistent even if the
* upstream ref moves mid-sync.
*/
resolveCommit(): Promise<RepoCommit>;
/**
* Lists every entry beneath `pathPrefix` recursively, as of `commit`. Paths
* are returned relative to the repository root rather than to `pathPrefix`,
* so entries listed from several configured paths can be merged without
* ambiguity.
*/
fetchTreeEntries(commit: RepoCommit, params: FetchTreeEntriesParams): Promise<RepoTreeEntry[]>;
/**
* Fetches one file's raw bytes. The caller has already checked `entry.size`
* against the skill import limits, so implementations do not repeat that.
*/
fetchFileContent(commit: RepoCommit, entry: RepoTreeEntry): Promise<Buffer>;
}

View file

@ -0,0 +1,15 @@
/**
* Failure raised anywhere in a skill sync run repository access, skill
* preparation, or database reconciliation. `code` is persisted on the sync
* status document and surfaced through the admin API, so it must stay a stable
* machine-readable identifier rather than a message.
*/
export class SkillSyncError extends Error {
code: string;
constructor(code: string, message: string) {
super(message);
this.name = 'SkillSyncError';
this.code = code;
}
}

View file

@ -12,6 +12,7 @@ import type {
UpdateSkillResult,
UpsertSkillFileInput,
} from '@librechat/data-schemas';
import type { RepoTreeEntry, GitRepoAdapter } from './adapters/types';
import type { GitHubSkillSyncDeps } from './github';
import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits';
import { createGitHubSkillSyncRunner } from './github';
@ -3418,3 +3419,88 @@ describe('createGitHubSkillSyncRunner', () => {
}
});
});
describe('repository adapter seam', () => {
/** Stands in for any provider: a flat repository held in memory. */
function createFakeAdapter(files: Record<string, string>): GitRepoAdapter {
const entries: RepoTreeEntry[] = Object.entries(files).map(([path, content]) => ({
path,
type: 'blob',
id: `${path}@1`,
size: Buffer.byteLength(content),
}));
return {
resolveCommit: async () => ({ id: 'fake-commit', treeId: 'fake-tree' }),
fetchTreeEntries: async (_commit, { pathPrefix }) =>
entries.filter((entry) => !pathPrefix || entry.path.startsWith(`${pathPrefix}/`)),
fetchFileContent: async (_commit, entry) => Buffer.from(files[entry.path]),
};
}
it('publishes skills read through any repository client, with no provider requests', async () => {
const deps = createDeps({
createAdapter: () =>
createFakeAdapter({
'skills/research/SKILL.md':
'---\nname: research\ndescription: Research things\n---\nBody',
'skills/research/scripts/run.sh': 'echo hi',
}),
});
const result = await createGitHubSkillSyncRunner(deps).runOnce();
expect(result.status).toBe('completed');
expect(deps.fetchFn).not.toHaveBeenCalled();
expect(deps.createSkill).toHaveBeenCalledWith(
expect.objectContaining({
name: 'research',
sourceMetadata: expect.objectContaining({
commitSha: 'fake-commit',
skillBlobSha: 'skills/research/SKILL.md@1',
}),
}),
);
expect(deps.upsertSkillFile).toHaveBeenCalledWith(
expect.objectContaining({
relativePath: 'scripts/run.sh',
sourceMetadata: expect.objectContaining({
commitSha: 'fake-commit',
blobSha: 'skills/research/scripts/run.sh@1',
}),
}),
);
});
it('re-downloads a file only when the adapter reports a new content id', async () => {
const deps = createDeps({
createAdapter: () =>
createFakeAdapter({
'skills/research/SKILL.md':
'---\nname: research\ndescription: Research things\n---\nBody',
'skills/research/scripts/run.sh': 'echo hi',
}),
getSkillFileByPath: jest.fn(async () => ({
_id: new Types.ObjectId(),
skillId: new Types.ObjectId(),
relativePath: 'scripts/run.sh',
file_id: 'existing-file-id',
filename: 'run.sh',
filepath: '/uploads/existing-file-id__run.sh',
source: 'local',
sourceMetadata: { blobSha: 'skills/research/scripts/run.sh@1' },
mimeType: 'application/x-sh',
bytes: 7,
category: 'script' as const,
isExecutable: false,
author: new Types.ObjectId(),
createdAt: new Date(),
updatedAt: new Date(),
})),
});
const result = await createGitHubSkillSyncRunner(deps).runOnce();
expect(result.status).toBe('completed');
expect(deps.upsertSkillFile).not.toHaveBeenCalled();
});
});

View file

@ -24,10 +24,22 @@ import type {
SkillSyncStatusInput,
} from '@librechat/data-schemas';
import type { SkillSyncConfig, SkillSyncGitHubSourceConfig } from 'librechat-data-provider';
import type {
RepoCommit,
RepoTreeEntry,
GitRepoAdapter,
AssertNotCancelled,
} from './adapters/types';
import type { GitHubRepoAdapterConfig } from './adapters/github';
import {
GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION,
createGitHubRepoAdapter,
} from './adapters/github';
import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits';
import { parseSkillMarkdown } from '../parse';
import { normalizeRepoPath } from './path';
import { SkillSyncError } from './errors';
const GITHUB_API_BASE = 'https://api.github.com';
const SYSTEM_AUTHOR_NAME = 'GitHub Sync';
let systemAuthorId: Types.ObjectId | undefined;
@ -51,42 +63,11 @@ const VALIDATION_ISSUE_FIELD_MAX = 100;
const VALIDATION_ISSUE_CODE_MAX = 64;
const VALIDATION_ISSUE_MESSAGE_MAX = 250;
export const GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION =
'Use a GitHub fine-grained personal access token scoped to the selected repository with read-only Contents and Metadata permissions.';
export { GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION };
export type { GitHubRepoAdapterConfig };
type FetchFn = typeof fetch;
type GitHubTreeEntry = {
path: string;
mode: string;
type: 'blob' | 'tree' | 'commit';
sha: string;
size?: number;
url: string;
};
type GitHubTreeResponse = {
sha: string;
tree: GitHubTreeEntry[];
truncated: boolean;
};
type GitHubBlobResponse = {
sha: string;
content: string;
encoding: string;
size: number;
};
type GitHubCommitResponse = {
sha: string;
commit: {
tree: {
sha: string;
};
};
};
type SyncCounters = {
syncedSkillCount: number;
syncedFileCount: number;
@ -95,12 +76,10 @@ type SyncCounters = {
skippedSkillCount: number;
};
type AssertNotCancelled = () => void;
type DiscoveredSkill = {
rootPath: string;
skillMd: GitHubTreeEntry;
files: GitHubTreeEntry[];
skillMd: RepoTreeEntry;
files: RepoTreeEntry[];
};
type UpsertRemoteSkillResult = {
@ -239,6 +218,12 @@ export type GitHubSkillSyncDeps = {
grantedBy: string | Types.ObjectId;
}) => Promise<unknown>;
fetchFn?: FetchFn;
/**
* Builds the repository client a source is synced through. Defaults to the
* GitHub adapter; overridable so the orchestration can be exercised against a
* fake repository without standing up provider HTTP responses.
*/
createAdapter?: (config: GitHubRepoAdapterConfig) => GitRepoAdapter;
lockOwner?: string;
allowServerCredentials?: boolean;
};
@ -263,21 +248,6 @@ export type GitHubSkillSyncRunner = {
runOnce: () => Promise<GitHubSkillSyncRunResult>;
};
class SkillSyncError extends Error {
code: string;
constructor(code: string, message: string) {
super(message);
this.name = 'SkillSyncError';
this.code = code;
}
}
function normalizeRepoPath(value: string): string {
const trimmed = value.trim().replace(/^\/+|\/+$/g, '');
return trimmed === '.' ? '' : trimmed;
}
function isSafeRelativePath(value: string): boolean {
if (!value || value.startsWith('/') || value.startsWith('\\')) {
return false;
@ -382,7 +352,7 @@ function getLimitMegabytes(bytes: number): number {
return Math.round(bytes / 1024 / 1024);
}
function assertGitHubBlobSize(entry: GitHubTreeEntry, relativePath: string): number {
function assertGitHubBlobSize(entry: RepoTreeEntry, relativePath: string): number {
if (typeof entry.size !== 'number' || !Number.isFinite(entry.size) || entry.size < 0) {
throw new SkillSyncError(
'GITHUB_BLOB_SIZE_UNKNOWN',
@ -439,7 +409,7 @@ function getSkillMdPath(discovered: DiscoveredSkill): string {
return discovered.rootPath ? `${discovered.rootPath}/SKILL.md` : 'SKILL.md';
}
function getDiscoveredRelativePath(discovered: DiscoveredSkill, entry: GitHubTreeEntry): string {
function getDiscoveredRelativePath(discovered: DiscoveredSkill, entry: RepoTreeEntry): string {
const prefix = discovered.rootPath ? `${discovered.rootPath}/` : '';
const normalized = normalizeRepoPath(entry.path);
return prefix ? normalized.slice(prefix.length) : normalized;
@ -630,186 +600,23 @@ function truncateSkipName(name: string | undefined): string | undefined {
return sanitized.length > SKIP_NAME_MAX ? `${sanitized.slice(0, SKIP_NAME_MAX - 1)}` : sanitized;
}
function buildGitHubHeaders(token: string): HeadersInit {
return {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${token}`,
'X-GitHub-Api-Version': '2022-11-28',
'User-Agent': 'LibreChat-Skill-Sync',
};
}
function buildGitHubUrl(pathname: string): string {
return `${GITHUB_API_BASE}${pathname}`;
}
function encodeGitHubPath(value: string): string {
return value.split('/').map(encodeURIComponent).join('/');
}
async function readGitHubErrorMessage(response: Response): Promise<string | undefined> {
try {
const body = (await response.json()) as { message?: unknown };
return typeof body.message === 'string' ? body.message : undefined;
} catch {
return undefined;
}
}
function isGitHubRateLimitResponse(params: {
status: number;
remaining: string | null;
retryAfter: string | null;
message?: string;
}): boolean {
if (params.status === 429 || params.remaining === '0' || params.retryAfter) {
return true;
}
const message = params.message?.toLowerCase() ?? '';
return message.includes('rate limit') || message.includes('abuse detection');
}
async function githubJson<T>(params: {
fetchFn: FetchFn;
token: string;
pathname: string;
}): Promise<T> {
let response: Response;
try {
response = await params.fetchFn(buildGitHubUrl(params.pathname), {
headers: buildGitHubHeaders(params.token),
});
} catch {
throw new SkillSyncError(
'GITHUB_REQUEST_FAILED',
'GitHub request failed before receiving a response',
);
}
if (response.ok) {
return (await response.json()) as T;
}
const remaining = response.headers.get('x-ratelimit-remaining');
const retryAfter = response.headers.get('retry-after');
const message = await readGitHubErrorMessage(response);
if (response.status === 401 || response.status === 403 || response.status === 429) {
const code = isGitHubRateLimitResponse({
status: response.status,
remaining,
retryAfter,
message,
})
? 'GITHUB_RATE_LIMITED'
: 'GITHUB_AUTH_FAILED';
throw new SkillSyncError(code, `GitHub request failed with HTTP ${response.status}`);
}
if (response.status === 404) {
throw new SkillSyncError('GITHUB_NOT_FOUND', 'GitHub repository, ref, or path was not found');
}
throw new SkillSyncError(
'GITHUB_REQUEST_FAILED',
`GitHub request failed with HTTP ${response.status}`,
);
}
async function fetchCommit(params: {
fetchFn: FetchFn;
token: string;
source: SkillSyncGitHubSourceConfig;
}): Promise<GitHubCommitResponse> {
const owner = encodeURIComponent(params.source.owner);
const repo = encodeURIComponent(params.source.repo);
const ref = encodeGitHubPath(params.source.ref);
return githubJson<GitHubCommitResponse>({
fetchFn: params.fetchFn,
token: params.token,
pathname: `/repos/${owner}/${repo}/commits/${ref}`,
});
}
async function fetchTree(params: {
fetchFn: FetchFn;
token: string;
source: SkillSyncGitHubSourceConfig;
treeSha: string;
recursive?: boolean;
}): Promise<GitHubTreeResponse> {
const owner = encodeURIComponent(params.source.owner);
const repo = encodeURIComponent(params.source.repo);
const treeSha = encodeURIComponent(params.treeSha);
const recursive = params.recursive ?? true;
return githubJson<GitHubTreeResponse>({
fetchFn: params.fetchFn,
token: params.token,
pathname: `/repos/${owner}/${repo}/git/trees/${treeSha}${recursive ? '?recursive=1' : ''}`,
});
}
async function fetchTreeAtPath(params: {
fetchFn: FetchFn;
token: string;
source: SkillSyncGitHubSourceConfig;
rootTreeSha: string;
repoPath: string;
assertNotCancelled: AssertNotCancelled;
}): Promise<GitHubTreeEntry[]> {
const normalizedPath = normalizeRepoPath(params.repoPath);
let treeSha = params.rootTreeSha;
if (normalizedPath) {
for (const segment of normalizedPath.split('/')) {
params.assertNotCancelled();
const tree = await fetchTree({
fetchFn: params.fetchFn,
token: params.token,
source: params.source,
treeSha,
recursive: false,
});
params.assertNotCancelled();
if (tree.truncated) {
throw new SkillSyncError('GITHUB_TREE_TRUNCATED', 'GitHub tree response was truncated');
}
const next = tree.tree.find((entry) => entry.type === 'tree' && entry.path === segment);
if (!next) {
throw new SkillSyncError(
'GITHUB_PATH_NOT_FOUND',
`Configured GitHub skill path "${normalizedPath}" was not found`,
);
}
treeSha = next.sha;
}
}
params.assertNotCancelled();
const tree = await fetchTree({
fetchFn: params.fetchFn,
token: params.token,
source: params.source,
treeSha,
recursive: true,
});
params.assertNotCancelled();
if (tree.truncated) {
throw new SkillSyncError('GITHUB_TREE_TRUNCATED', 'GitHub tree response was truncated');
}
if (!normalizedPath) {
return tree.tree;
}
return tree.tree.map((entry) => ({
...entry,
path: `${normalizedPath}/${normalizeRepoPath(entry.path)}`,
}));
}
/**
* Merges the recursive listings of every configured path into one entry set.
* Configured paths may nest, so entries are deduplicated on their normalized
* repository path rather than concatenated.
*/
async function fetchConfiguredTreeEntries(params: {
fetchFn: FetchFn;
token: string;
adapter: GitRepoAdapter;
commit: RepoCommit;
source: SkillSyncGitHubSourceConfig;
rootTreeSha: string;
assertNotCancelled: AssertNotCancelled;
}): Promise<GitHubTreeEntry[]> {
const entriesByPath = new Map<string, GitHubTreeEntry>();
}): Promise<RepoTreeEntry[]> {
const entriesByPath = new Map<string, RepoTreeEntry>();
for (const repoPath of params.source.paths) {
const entries = await fetchTreeAtPath({ ...params, repoPath });
const entries = await params.adapter.fetchTreeEntries(params.commit, {
pathPrefix: repoPath,
assertNotCancelled: params.assertNotCancelled,
});
for (const entry of entries) {
const normalizedPath = normalizeRepoPath(entry.path);
entriesByPath.set(normalizedPath, { ...entry, path: normalizedPath });
@ -818,29 +625,6 @@ async function fetchConfiguredTreeEntries(params: {
return [...entriesByPath.values()];
}
async function fetchBlob(params: {
fetchFn: FetchFn;
token: string;
source: SkillSyncGitHubSourceConfig;
sha: string;
}): Promise<Buffer> {
const owner = encodeURIComponent(params.source.owner);
const repo = encodeURIComponent(params.source.repo);
const sha = encodeURIComponent(params.sha);
const blob = await githubJson<GitHubBlobResponse>({
fetchFn: params.fetchFn,
token: params.token,
pathname: `/repos/${owner}/${repo}/git/blobs/${sha}`,
});
if (blob.encoding !== 'base64') {
throw new SkillSyncError(
'GITHUB_UNSUPPORTED_BLOB',
`Unsupported GitHub blob encoding "${blob.encoding}"`,
);
}
return Buffer.from(blob.content.replace(/\s/g, ''), 'base64');
}
function isSkillRootWithinDiscoveryDepth(
rootPath: string,
basePath: string,
@ -860,12 +644,12 @@ function isSkillRootWithinDiscoveryDepth(
}
function discoverSkills(
tree: GitHubTreeEntry[],
tree: RepoTreeEntry[],
source: SkillSyncGitHubSourceConfig,
): DiscoveredSkill[] {
const basePaths = source.paths.map(normalizeRepoPath);
const skillDiscoveryDepth = source.skillDiscoveryDepth ?? SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH;
const skillMdByRoot = new Map<string, GitHubTreeEntry>();
const skillMdByRoot = new Map<string, RepoTreeEntry>();
for (const entry of tree) {
if (entry.type !== 'blob') {
continue;
@ -911,7 +695,7 @@ function discoverSkills(
}
function assertConfiguredPathsExist(
tree: GitHubTreeEntry[],
tree: RepoTreeEntry[],
source: SkillSyncGitHubSourceConfig,
): void {
for (const configuredPath of source.paths.map(normalizeRepoPath)) {
@ -1015,7 +799,7 @@ async function prepareRemoteSkill(params: {
ref: source.ref,
skillPath: discovered.rootPath,
commitSha,
skillBlobSha: discovered.skillMd.sha,
skillBlobSha: discovered.skillMd.id,
syncedAt: serializeDate(syncedAt),
syncStatus: 'synced',
};
@ -1485,16 +1269,15 @@ async function deleteNameConflictingStaleSkill(params: {
async function syncSkillFiles(params: {
deps: GitHubSkillSyncDeps;
token: string;
adapter: GitRepoAdapter;
commit: RepoCommit;
source: SkillSyncGitHubSourceConfig;
skill: ISkill & { _id: Types.ObjectId };
discovered: DiscoveredSkill;
commitSha: string;
fetchFn: FetchFn;
assertNotCancelled: AssertNotCancelled;
journal?: SyncSkillFilesJournal;
}): Promise<SyncSkillFilesResult> {
const { deps, token, source, skill, discovered, commitSha, fetchFn, assertNotCancelled } = params;
const { deps, adapter, commit, source, skill, discovered, assertNotCancelled } = params;
const journal = params.journal ?? { staleFiles: [], savedFiles: [] };
const remotePaths = new Set<string>();
let syncedFileCount = 0;
@ -1511,10 +1294,10 @@ async function syncSkillFiles(params: {
assertCumulativeGitHubFileSize(totalFileBytes);
remotePaths.add(relativePath);
const existing = await deps.getSkillFileByPath(skill._id, relativePath);
if (existing && getSourceMetadataString(existing, 'blobSha') === entry.sha) {
if (existing && getSourceMetadataString(existing, 'blobSha') === entry.id) {
continue;
}
const buffer = await fetchBlob({ fetchFn, token, source, sha: entry.sha });
const buffer = await adapter.fetchFileContent(commit, entry);
assertNotCancelled();
assertGitHubBufferSize(buffer, relativePath);
const fileId = crypto.randomUUID();
@ -1543,8 +1326,8 @@ async function syncSkillFiles(params: {
provider: PROVIDER,
sourceId: source.id,
upstreamId: makeUpstreamId(source, discovered.rootPath),
commitSha,
blobSha: entry.sha,
commitSha: commit.id,
blobSha: entry.id,
path: entry.path,
},
mimeType,
@ -1646,6 +1429,7 @@ async function syncSource(params: {
assertNotCancelled: AssertNotCancelled;
}): Promise<ISkillSyncStatus> {
const { deps, source, fetchFn, assertNotCancelled } = params;
const createAdapter = deps.createAdapter ?? createGitHubRepoAdapter;
const startedAt = new Date();
const counts: SyncCounters = {
syncedSkillCount: 0,
@ -1667,13 +1451,13 @@ async function syncSource(params: {
getMissingCredentialMessage(source, allowServerCredentials),
);
}
const commit = await fetchCommit({ fetchFn, token, source });
const adapter = createAdapter({ source, token, fetchFn });
const commit = await adapter.resolveCommit();
assertNotCancelled();
const treeEntries = await fetchConfiguredTreeEntries({
fetchFn,
token,
adapter,
commit,
source,
rootTreeSha: commit.commit.tree.sha,
assertNotCancelled,
});
assertConfiguredPathsExist(treeEntries, source);
@ -1777,12 +1561,7 @@ async function syncSource(params: {
try {
assertGitHubSkillPackageManifest(discovered);
const skillMdPath = getSkillMdPath(discovered);
const skillMdBuffer = await fetchBlob({
fetchFn,
token,
source,
sha: discovered.skillMd.sha,
});
const skillMdBuffer = await adapter.fetchFileContent(commit, discovered.skillMd);
assertNotCancelled();
assertGitHubBufferSize(skillMdBuffer, skillMdPath);
const prepared = await prepareRemoteSkill({
@ -1790,7 +1569,7 @@ async function syncSource(params: {
source,
discovered,
skillMdContent: skillMdBuffer.toString('utf-8'),
commitSha: commit.sha,
commitSha: commit.id,
syncedAt,
});
preparedSkills.push({ discovered, prepared });
@ -1907,12 +1686,11 @@ async function syncSource(params: {
try {
fileCounts = await syncSkillFiles({
deps,
token,
adapter,
commit,
source,
skill: effectivePrepared.existing,
discovered,
commitSha: commit.sha,
fetchFn,
assertNotCancelled,
journal,
});
@ -1990,12 +1768,11 @@ async function syncSource(params: {
try {
const fileCounts = await syncSkillFiles({
deps,
token,
adapter,
commit,
source,
skill,
discovered,
commitSha: commit.sha,
fetchFn,
assertNotCancelled,
});
await ensurePublicViewer(deps, skill._id);

View file

@ -1,3 +1,4 @@
export * from './adapters/types';
export * from './github';
export * from './orchestrator';
export * from './scheduler';

View file

@ -0,0 +1,10 @@
/**
* Repository paths arrive from provider APIs and admin config with inconsistent
* leading/trailing slashes, and `.` for the repository root. Normalizing both
* sides through this keeps configured paths, tree entry paths, and stored skill
* paths directly comparable.
*/
export function normalizeRepoPath(value: string): string {
const trimmed = value.trim().replace(/^\/+|\/+$/g, '');
return trimmed === '.' ? '' : trimmed;
}