mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-16 13:00:55 +00:00
perf: Skip Unchanged Skills in GitHub Skill Sync
Each mirrored skill now records a sync fingerprint in its sourceMetadata, hashing the mapping version, the upstream identity (source coordinates, skill path and SKILL.md blob id) and the stored definition. When a later run finds the fingerprint still matching and the stored files matching the discovered blobs, it neither downloads SKILL.md nor re-reads and updates the skill; the public viewer grant, name-conflict cleanup, stale-skill reconciliation and status counts run as before. Mirrors without a fingerprint take the full path once and record it.
This commit is contained in:
parent
97bccd2e7d
commit
565976be52
2 changed files with 581 additions and 45 deletions
|
|
@ -1,6 +1,7 @@
|
|||
import crypto from 'crypto';
|
||||
import { Types } from 'mongoose';
|
||||
import { logger, getTenantId } from '@librechat/data-schemas';
|
||||
import mongoose, { Types } from 'mongoose';
|
||||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { logger, getTenantId, createModels, createMethods } from '@librechat/data-schemas';
|
||||
import type {
|
||||
ISkill,
|
||||
ISkillFile,
|
||||
|
|
@ -2861,7 +2862,7 @@ describe('createGitHubSkillSyncRunner', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('skips existing skill updates when the upstream package is unchanged', async () => {
|
||||
it('records a sync fingerprint once on an unchanged mirror that predates it', async () => {
|
||||
const skillMarkdown = '---\nname: research\ndescription: Research things\n---\nBody';
|
||||
const existing = makeSkill({
|
||||
name: 'research',
|
||||
|
|
@ -2901,7 +2902,15 @@ describe('createGitHubSkillSyncRunner', () => {
|
|||
getSkillById: jest.fn(async () => existing),
|
||||
getSkillFileByPath: jest.fn(async () => unchangedFile),
|
||||
listSkillFiles: jest.fn(async () => [unchangedFile]),
|
||||
updateSkill: jest.fn(),
|
||||
updateSkill: jest.fn(
|
||||
async ({ update }: { update: UpdateSkillInput }): Promise<UpdateSkillResult> => ({
|
||||
status: 'updated',
|
||||
skill: { ...existing, ...update, version: existing.version + 1 } as ISkill & {
|
||||
_id: Types.ObjectId;
|
||||
},
|
||||
warnings: [],
|
||||
}),
|
||||
),
|
||||
});
|
||||
const runner = createGitHubSkillSyncRunner(deps);
|
||||
const result = await runner.runOnce();
|
||||
|
|
@ -2909,7 +2918,18 @@ describe('createGitHubSkillSyncRunner', () => {
|
|||
expect(result.status).toBe('completed');
|
||||
expect(deps.saveBuffer).not.toHaveBeenCalled();
|
||||
expect(deps.upsertSkillFile).not.toHaveBeenCalled();
|
||||
expect(deps.updateSkill).not.toHaveBeenCalled();
|
||||
expect(deps.updateSkill).toHaveBeenCalledTimes(1);
|
||||
expect(deps.updateSkill).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
update: expect.objectContaining({
|
||||
body: skillMarkdown,
|
||||
sourceMetadata: expect.objectContaining({
|
||||
skillBlobSha: 'skill-md-sha',
|
||||
syncFingerprint: expect.stringMatching(/^[0-9a-f]{64}$/),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(deps.grantPermission).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -3732,3 +3752,320 @@ describe('files whose paths cannot be mirrored', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('steady-state sync of an unchanged source', () => {
|
||||
const researchMarkdown = '---\nname: research\ndescription: Research things\n---\nBody';
|
||||
const writingMarkdown = '---\nname: writing\ndescription: Write things\n---\nBody';
|
||||
let server: MongoMemoryServer;
|
||||
let db: ReturnType<typeof createMethods>;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await MongoMemoryServer.create();
|
||||
await mongoose.connect(server.getUri());
|
||||
createModels(mongoose);
|
||||
db = createMethods(mongoose);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await server.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await mongoose.connection.dropDatabase();
|
||||
});
|
||||
|
||||
type RepoFile = { content: string; revision?: number };
|
||||
|
||||
/** The provider boundary: a repository held in memory whose content ids can move. */
|
||||
function createRepo(initial: Record<string, RepoFile>) {
|
||||
let files = initial;
|
||||
const fetchFileContent = jest.fn(async (_commit: unknown, entry: RepoTreeEntry) =>
|
||||
Buffer.from(files[entry.path].content),
|
||||
);
|
||||
const adapter: GitRepoAdapter = {
|
||||
resolveCommit: async () => ({ id: 'fake-commit', treeId: 'fake-tree' }),
|
||||
fetchTreeEntries: async (_commit, { pathPrefix }) =>
|
||||
Object.entries(files)
|
||||
.map(([path, file]) => ({
|
||||
path,
|
||||
type: 'blob' as const,
|
||||
id: `${path}@${file.revision ?? 1}`,
|
||||
size: Buffer.byteLength(file.content),
|
||||
}))
|
||||
.filter((entry) => !pathPrefix || entry.path.startsWith(`${pathPrefix}/`)),
|
||||
fetchFileContent,
|
||||
};
|
||||
return {
|
||||
adapter,
|
||||
fetchFileContent,
|
||||
setFiles: (next: Record<string, RepoFile>) => {
|
||||
files = next;
|
||||
},
|
||||
fetchedPaths: () => fetchFileContent.mock.calls.map(([, entry]) => entry.path),
|
||||
};
|
||||
}
|
||||
|
||||
function createDbDeps(
|
||||
repo: ReturnType<typeof createRepo>,
|
||||
sourceOverrides: Partial<{ ref: string }> = {},
|
||||
) {
|
||||
return createDeps({
|
||||
getConfig: () => ({
|
||||
github: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
runOnStartup: false,
|
||||
sources: [
|
||||
{
|
||||
id: 'librechat-skills',
|
||||
owner: 'LibreChat',
|
||||
repo: 'skills',
|
||||
ref: sourceOverrides.ref ?? 'main',
|
||||
paths: ['skills'],
|
||||
credentialKey: 'github-skills-prod',
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
createAdapter: () => repo.adapter,
|
||||
createSkill: jest.fn(db.createSkill),
|
||||
updateSkill: jest.fn(db.updateSkill),
|
||||
getSkillById: jest.fn(db.getSkillById),
|
||||
findSkillBySourceIdentity: jest.fn(db.findSkillBySourceIdentity),
|
||||
listSkillsBySource: jest.fn(db.listSkillsBySource),
|
||||
listSkillFiles: jest.fn(db.listSkillFiles),
|
||||
getSkillFileByPath: jest.fn(db.getSkillFileByPath),
|
||||
upsertSkillFile: jest.fn(db.upsertSkillFile),
|
||||
deleteSkillFile: jest.fn(db.deleteSkillFile),
|
||||
deleteSkill: jest.fn(db.deleteSkill),
|
||||
saveBuffer: jest.fn(async ({ fileName }: { fileName: string }) => ({
|
||||
filepath: `/uploads/${fileName}`,
|
||||
source: 'local',
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
async function seed(repo: ReturnType<typeof createRepo>) {
|
||||
const result = await createGitHubSkillSyncRunner(createDbDeps(repo)).runOnce();
|
||||
expect(result.status).toBe('completed');
|
||||
expect(result.sources[0]).toEqual(
|
||||
expect.objectContaining({ status: 'succeeded', syncedSkillCount: expect.any(Number) }),
|
||||
);
|
||||
repo.fetchFileContent.mockClear();
|
||||
}
|
||||
|
||||
async function getMirror(rootPath = 'skills/research') {
|
||||
const skill = await db.findSkillBySourceIdentity({
|
||||
source: 'github',
|
||||
upstreamId: `librechat-skills:${rootPath}`,
|
||||
});
|
||||
if (!skill) {
|
||||
throw new Error(`missing mirror for ${rootPath}`);
|
||||
}
|
||||
return skill;
|
||||
}
|
||||
|
||||
const baseRepo = (): Record<string, RepoFile> => ({
|
||||
'skills/research/SKILL.md': { content: researchMarkdown },
|
||||
'skills/research/scripts/run.sh': { content: 'echo hi' },
|
||||
'skills/research/notes.md': { content: 'notes' },
|
||||
});
|
||||
|
||||
it('neither downloads SKILL.md nor updates the skill on a second sync', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
const before = await getMirror();
|
||||
expect(before.sourceMetadata?.syncFingerprint).toEqual(expect.stringMatching(/^[0-9a-f]{64}$/));
|
||||
|
||||
const deps = createDbDeps(repo);
|
||||
const result = await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(result.sources[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
status: 'succeeded',
|
||||
syncedSkillCount: 1,
|
||||
syncedFileCount: 0,
|
||||
deletedSkillCount: 0,
|
||||
deletedFileCount: 0,
|
||||
}),
|
||||
);
|
||||
expect(repo.fetchFileContent).not.toHaveBeenCalled();
|
||||
expect(deps.updateSkill).not.toHaveBeenCalled();
|
||||
expect(deps.getSkillFileByPath).not.toHaveBeenCalled();
|
||||
expect(deps.upsertSkillFile).not.toHaveBeenCalled();
|
||||
expect(deps.grantPermission).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ resourceId: before._id }),
|
||||
);
|
||||
expect((await getMirror()).version).toBe(before.version);
|
||||
});
|
||||
|
||||
it('downloads SKILL.md and updates the skill when its blob changes', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
repo.setFiles({
|
||||
...baseRepo(),
|
||||
'skills/research/SKILL.md': {
|
||||
content: '---\nname: research\ndescription: Research more things\n---\nBody',
|
||||
revision: 2,
|
||||
},
|
||||
});
|
||||
|
||||
const deps = createDbDeps(repo);
|
||||
await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(repo.fetchedPaths()).toEqual(['skills/research/SKILL.md']);
|
||||
expect(deps.updateSkill).toHaveBeenCalledTimes(1);
|
||||
const mirror = await getMirror();
|
||||
expect(mirror.description).toBe('Research more things');
|
||||
expect(mirror.sourceMetadata?.skillBlobSha).toBe('skills/research/SKILL.md@2');
|
||||
});
|
||||
|
||||
it('updates the skill when a bundled file changes', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
repo.setFiles({
|
||||
...baseRepo(),
|
||||
'skills/research/scripts/run.sh': { content: 'echo bye', revision: 2 },
|
||||
});
|
||||
|
||||
const deps = createDbDeps(repo);
|
||||
const result = await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(repo.fetchedPaths().sort()).toEqual([
|
||||
'skills/research/SKILL.md',
|
||||
'skills/research/scripts/run.sh',
|
||||
]);
|
||||
expect(deps.updateSkill).toHaveBeenCalledTimes(1);
|
||||
expect(result.sources[0]).toEqual(expect.objectContaining({ syncedFileCount: 1 }));
|
||||
const mirror = await getMirror();
|
||||
const file = await db.getSkillFileByPath(mirror._id, 'scripts/run.sh');
|
||||
expect(file?.sourceMetadata?.blobSha).toBe('skills/research/scripts/run.sh@2');
|
||||
});
|
||||
|
||||
it('updates the skill and deletes the file when a bundled file is removed upstream', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
const { 'skills/research/notes.md': _removed, ...remaining } = baseRepo();
|
||||
repo.setFiles(remaining);
|
||||
|
||||
const deps = createDbDeps(repo);
|
||||
const result = await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(repo.fetchedPaths()).toEqual(['skills/research/SKILL.md']);
|
||||
expect(deps.updateSkill).toHaveBeenCalledTimes(1);
|
||||
expect(deps.deleteSkillFile).toHaveBeenCalledWith(expect.anything(), 'notes.md');
|
||||
expect(result.sources[0]).toEqual(expect.objectContaining({ deletedFileCount: 1 }));
|
||||
const files = await db.listSkillFiles((await getMirror())._id);
|
||||
expect(files.map((file) => file.relativePath)).toEqual(['scripts/run.sh']);
|
||||
});
|
||||
|
||||
it('updates the skill when the source config it records changes', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
|
||||
const deps = createDbDeps(repo, { ref: 'release' });
|
||||
await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(repo.fetchedPaths()).toEqual(['skills/research/SKILL.md']);
|
||||
expect(deps.updateSkill).toHaveBeenCalledTimes(1);
|
||||
expect((await getMirror()).sourceMetadata?.ref).toBe('release');
|
||||
});
|
||||
|
||||
it('re-syncs a mirror whose stored definition was edited outside the sync', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
const mirror = await getMirror();
|
||||
await db.updateSkill({
|
||||
id: mirror._id.toString(),
|
||||
expectedVersion: mirror.version,
|
||||
update: { body: 'Edited body' },
|
||||
});
|
||||
|
||||
const deps = createDbDeps(repo);
|
||||
await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(repo.fetchedPaths()).toEqual(['skills/research/SKILL.md']);
|
||||
expect(deps.updateSkill).toHaveBeenCalledTimes(1);
|
||||
expect((await getMirror()).body).toBe(researchMarkdown);
|
||||
});
|
||||
|
||||
it('fully syncs a mirror without a fingerprint once, then skips it', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
const mirror = await getMirror();
|
||||
const { syncFingerprint: _fingerprint, ...legacyMetadata } = mirror.sourceMetadata ?? {};
|
||||
await db.updateSkill({
|
||||
id: mirror._id.toString(),
|
||||
expectedVersion: mirror.version,
|
||||
update: { sourceMetadata: legacyMetadata },
|
||||
});
|
||||
|
||||
const legacyDeps = createDbDeps(repo);
|
||||
await createGitHubSkillSyncRunner(legacyDeps).runOnce();
|
||||
|
||||
expect(repo.fetchedPaths()).toEqual(['skills/research/SKILL.md']);
|
||||
expect(legacyDeps.updateSkill).toHaveBeenCalledTimes(1);
|
||||
expect(legacyDeps.upsertSkillFile).not.toHaveBeenCalled();
|
||||
expect((await getMirror()).sourceMetadata?.syncFingerprint).toBe(
|
||||
mirror.sourceMetadata?.syncFingerprint,
|
||||
);
|
||||
|
||||
repo.fetchFileContent.mockClear();
|
||||
const steadyDeps = createDbDeps(repo);
|
||||
await createGitHubSkillSyncRunner(steadyDeps).runOnce();
|
||||
|
||||
expect(repo.fetchFileContent).not.toHaveBeenCalled();
|
||||
expect(steadyDeps.updateSkill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still reconciles a skill deleted upstream while skipping the unchanged ones', async () => {
|
||||
const repo = createRepo({
|
||||
...baseRepo(),
|
||||
'skills/writing/SKILL.md': { content: writingMarkdown },
|
||||
'skills/writing/guide.md': { content: 'guide' },
|
||||
});
|
||||
await seed(repo);
|
||||
const writing = await getMirror('skills/writing');
|
||||
repo.setFiles(baseRepo());
|
||||
|
||||
const deps = createDbDeps(repo);
|
||||
const result = await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(repo.fetchFileContent).not.toHaveBeenCalled();
|
||||
expect(deps.updateSkill).not.toHaveBeenCalled();
|
||||
expect(deps.deleteSkill).toHaveBeenCalledWith(writing._id.toString());
|
||||
expect(result.sources[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
status: 'succeeded',
|
||||
syncedSkillCount: 1,
|
||||
deletedSkillCount: 1,
|
||||
deletedFileCount: 1,
|
||||
}),
|
||||
);
|
||||
expect(await db.getSkillById(writing._id)).toBeNull();
|
||||
});
|
||||
|
||||
it('still rejects an unchanged skill that exceeds the import limits', async () => {
|
||||
const repo = createRepo(baseRepo());
|
||||
await seed(repo);
|
||||
const maxEntries = DEFAULT_SKILL_IMPORT_LIMITS.maxEntries;
|
||||
const tooMany = { ...baseRepo() };
|
||||
for (let index = 0; index < maxEntries; index++) {
|
||||
tooMany[`skills/research/extra/${index}.txt`] = { content: 'x' };
|
||||
}
|
||||
repo.setFiles(tooMany);
|
||||
|
||||
const deps = createDbDeps(repo);
|
||||
const result = await createGitHubSkillSyncRunner(deps).runOnce();
|
||||
|
||||
expect(result.sources[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
skippedSkillCount: 1,
|
||||
skippedSkills: [expect.objectContaining({ errorCode: 'GITHUB_TOO_MANY_FILES' })],
|
||||
}),
|
||||
);
|
||||
expect(deps.findSkillBySourceIdentity).not.toHaveBeenCalled();
|
||||
expect(repo.fetchFileContent).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -69,6 +69,12 @@ const VALIDATION_ISSUE_LIMIT = 5;
|
|||
const VALIDATION_ISSUE_FIELD_MAX = 100;
|
||||
const VALIDATION_ISSUE_CODE_MAX = 64;
|
||||
const VALIDATION_ISSUE_MESSAGE_MAX = 250;
|
||||
/**
|
||||
* Bump when the mapping from an upstream package to a stored skill changes
|
||||
* (parsing, name fallback, frontmatter cleanup, metadata shape), so every
|
||||
* mirror takes the full sync path once and records the new fingerprint.
|
||||
*/
|
||||
const SYNC_FINGERPRINT_VERSION = 1;
|
||||
|
||||
export { GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION };
|
||||
export type { GitHubRepoAdapterConfig };
|
||||
|
|
@ -116,6 +122,8 @@ type PreparedExistingRemoteSkill = PreparedRemoteSkill & {
|
|||
type PreparedDiscoveredSkill = {
|
||||
discovered: DiscoveredSkill;
|
||||
prepared: PreparedRemoteSkill;
|
||||
/** The mirror already matches upstream, so syncing it writes nothing. */
|
||||
unchanged?: boolean;
|
||||
};
|
||||
|
||||
type SaveBufferResult = {
|
||||
|
|
@ -793,15 +801,181 @@ async function ensurePublicViewer(
|
|||
});
|
||||
}
|
||||
|
||||
async function prepareRemoteSkill(params: {
|
||||
type SkillDefinition = Pick<
|
||||
UpdateSkillInput,
|
||||
'name' | 'description' | 'body' | 'frontmatter' | 'alwaysApply'
|
||||
>;
|
||||
|
||||
/**
|
||||
* Source metadata that identifies what a mirror was synced from, minus the
|
||||
* per-run `commitSha` and `syncedAt`. Key order matches the stored shape.
|
||||
*/
|
||||
function makeSyncIdentity(
|
||||
source: SkillSyncGitHubSourceConfig,
|
||||
discovered: DiscoveredSkill,
|
||||
): Record<string, string> {
|
||||
return {
|
||||
provider: PROVIDER,
|
||||
sourceId: source.id,
|
||||
upstreamId: makeUpstreamId(source, discovered.rootPath),
|
||||
owner: source.owner,
|
||||
repo: source.repo,
|
||||
ref: source.ref,
|
||||
skillPath: discovered.rootPath,
|
||||
skillBlobSha: discovered.skillMd.id,
|
||||
syncStatus: 'synced',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes everything that decides a mirror's stored skill row: the mapping
|
||||
* version, the upstream identity (including the SKILL.md blob id), and the
|
||||
* definition written from it. Recomputed from the stored row, it only matches
|
||||
* when neither upstream nor the row changed since the last write.
|
||||
*/
|
||||
function makeSyncFingerprint(
|
||||
identity: Record<string, string>,
|
||||
definition: SkillDefinition,
|
||||
): string {
|
||||
const payload = JSON.stringify({
|
||||
version: SYNC_FINGERPRINT_VERSION,
|
||||
identity,
|
||||
name: definition.name,
|
||||
description: definition.description,
|
||||
body: definition.body,
|
||||
frontmatter: definition.frontmatter ?? {},
|
||||
alwaysApply: definition.alwaysApply ?? false,
|
||||
});
|
||||
return crypto.createHash('sha256').update(payload).digest('hex');
|
||||
}
|
||||
|
||||
function makeCreateInput(
|
||||
source: SkillSyncGitHubSourceConfig,
|
||||
update: UpdateSkillInput,
|
||||
fallbackName: string,
|
||||
): CreateSkillInput {
|
||||
return {
|
||||
...(update as Omit<UpdateSkillInput, 'source'>),
|
||||
name: update.name ?? fallbackName,
|
||||
description: update.description ?? fallbackName,
|
||||
author: makeSourceAuthorId(source),
|
||||
authorName: SYSTEM_AUTHOR_NAME,
|
||||
source: PROVIDER,
|
||||
tenantId: source.tenantId,
|
||||
};
|
||||
}
|
||||
|
||||
function getFallbackSkillName(
|
||||
source: SkillSyncGitHubSourceConfig,
|
||||
discovered: DiscoveredSkill,
|
||||
): string {
|
||||
return toSkillName(path.posix.basename(discovered.rootPath) || source.id);
|
||||
}
|
||||
|
||||
async function findExistingRemoteSkill(
|
||||
deps: GitHubSkillSyncDeps,
|
||||
source: SkillSyncGitHubSourceConfig,
|
||||
discovered: DiscoveredSkill,
|
||||
): Promise<(ISkill & { _id: Types.ObjectId }) | null> {
|
||||
const sourceTenantId = source.tenantId ?? undefined;
|
||||
const foundExisting = await deps.findSkillBySourceIdentity({
|
||||
source: PROVIDER,
|
||||
upstreamId: makeUpstreamId(source, discovered.rootPath),
|
||||
tenantId: sourceTenantId,
|
||||
});
|
||||
return foundExisting && (foundExisting.tenantId ?? undefined) === sourceTenantId
|
||||
? foundExisting
|
||||
: null;
|
||||
}
|
||||
|
||||
function hasSameSyncMetadata(
|
||||
stored: Record<string, unknown> | undefined,
|
||||
expected: Record<string, string>,
|
||||
): boolean {
|
||||
const { commitSha: _commitSha, syncedAt: _syncedAt, ...comparable } = stored ?? {};
|
||||
const keys = new Set([...Object.keys(comparable), ...Object.keys(expected)]);
|
||||
for (const key of keys) {
|
||||
if (comparable[key] !== expected[key]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** True when `syncSkillFiles` would neither download, replace nor delete a file. */
|
||||
function hasSameSyncedFiles(
|
||||
discovered: DiscoveredSkill,
|
||||
storedFiles: Array<ISkillFile & { _id: Types.ObjectId }>,
|
||||
): boolean {
|
||||
const remoteBlobShas = new Map<string, string>();
|
||||
for (const entry of discovered.files) {
|
||||
const relativePath = getDiscoveredRelativePath(discovered, entry);
|
||||
if (!isSafeRelativePath(relativePath) || relativePath.toUpperCase() === 'SKILL.MD') {
|
||||
continue;
|
||||
}
|
||||
remoteBlobShas.set(relativePath, entry.id);
|
||||
}
|
||||
if (storedFiles.length !== remoteBlobShas.size) {
|
||||
return false;
|
||||
}
|
||||
return storedFiles.every(
|
||||
(file) =>
|
||||
remoteBlobShas.has(file.relativePath) &&
|
||||
getSourceMetadataString(file, 'blobSha') === remoteBlobShas.get(file.relativePath),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a prepared skill without downloading SKILL.md when the mirror is
|
||||
* already current: its stored fingerprint matches the discovered upstream
|
||||
* package and its own definition, and its stored files match the discovered
|
||||
* blobs. Anything else returns null and takes the full sync path, which also
|
||||
* records the fingerprint on mirrors that predate it.
|
||||
*/
|
||||
async function prepareUnchangedRemoteSkill(params: {
|
||||
deps: GitHubSkillSyncDeps;
|
||||
source: SkillSyncGitHubSourceConfig;
|
||||
discovered: DiscoveredSkill;
|
||||
existing: ISkill & { _id: Types.ObjectId };
|
||||
}): Promise<PreparedExistingRemoteSkill | null> {
|
||||
const { deps, source, discovered, existing } = params;
|
||||
const identity = makeSyncIdentity(source, discovered);
|
||||
const definition: SkillDefinition = {
|
||||
name: existing.name,
|
||||
description: existing.description,
|
||||
body: existing.body,
|
||||
frontmatter: existing.frontmatter,
|
||||
alwaysApply: existing.alwaysApply,
|
||||
};
|
||||
const syncFingerprint = makeSyncFingerprint(identity, definition);
|
||||
if (!hasSameSyncMetadata(existing.sourceMetadata, { ...identity, syncFingerprint })) {
|
||||
return null;
|
||||
}
|
||||
const storedFiles = await deps.listSkillFiles(existing._id);
|
||||
if (!hasSameSyncedFiles(discovered, storedFiles)) {
|
||||
return null;
|
||||
}
|
||||
const update: UpdateSkillInput = {
|
||||
...definition,
|
||||
source: PROVIDER,
|
||||
sourceMetadata: existing.sourceMetadata,
|
||||
};
|
||||
return {
|
||||
existing,
|
||||
update,
|
||||
createInput: makeCreateInput(source, update, getFallbackSkillName(source, discovered)),
|
||||
};
|
||||
}
|
||||
|
||||
async function prepareRemoteSkill(params: {
|
||||
source: SkillSyncGitHubSourceConfig;
|
||||
discovered: DiscoveredSkill;
|
||||
existing: (ISkill & { _id: Types.ObjectId }) | null;
|
||||
skillMdContent: string;
|
||||
commitSha: string;
|
||||
syncedAt: Date;
|
||||
}): Promise<PreparedRemoteSkill> {
|
||||
const { deps, source, discovered, skillMdContent, commitSha, syncedAt } = params;
|
||||
const { source, discovered, existing, skillMdContent, commitSha, syncedAt } = params;
|
||||
const parsed = parseSkillMarkdown(skillMdContent);
|
||||
if (parsed.parseError) {
|
||||
throw new SkillSyncError(
|
||||
|
|
@ -815,50 +989,31 @@ async function prepareRemoteSkill(params: {
|
|||
`${discovered.rootPath}/SKILL.md contains invalid boolean frontmatter`,
|
||||
);
|
||||
}
|
||||
const upstreamId = makeUpstreamId(source, discovered.rootPath);
|
||||
const fallbackName = toSkillName(path.posix.basename(discovered.rootPath) || source.id);
|
||||
const sourceMetadata = {
|
||||
provider: PROVIDER,
|
||||
sourceId: source.id,
|
||||
upstreamId,
|
||||
owner: source.owner,
|
||||
repo: source.repo,
|
||||
ref: source.ref,
|
||||
skillPath: discovered.rootPath,
|
||||
commitSha,
|
||||
skillBlobSha: discovered.skillMd.id,
|
||||
syncedAt: serializeDate(syncedAt),
|
||||
syncStatus: 'synced',
|
||||
};
|
||||
const update: UpdateSkillInput = {
|
||||
const fallbackName = getFallbackSkillName(source, discovered);
|
||||
const definition: SkillDefinition = {
|
||||
name: parsed.name || fallbackName,
|
||||
description: parsed.description || parsed.name || fallbackName,
|
||||
body: skillMdContent,
|
||||
frontmatter: toCleanFrontmatter(parsed.frontmatter),
|
||||
alwaysApply: parsed.alwaysApply,
|
||||
source: PROVIDER,
|
||||
sourceMetadata,
|
||||
};
|
||||
const sourceTenantId = source.tenantId ?? undefined;
|
||||
const foundExisting = await deps.findSkillBySourceIdentity({
|
||||
source: PROVIDER,
|
||||
upstreamId,
|
||||
tenantId: sourceTenantId,
|
||||
});
|
||||
const existing =
|
||||
foundExisting && (foundExisting.tenantId ?? undefined) === sourceTenantId
|
||||
? foundExisting
|
||||
: null;
|
||||
const createInput: CreateSkillInput = {
|
||||
...(update as Omit<UpdateSkillInput, 'source'>),
|
||||
name: update.name ?? fallbackName,
|
||||
description: update.description ?? fallbackName,
|
||||
author: makeSourceAuthorId(source),
|
||||
authorName: SYSTEM_AUTHOR_NAME,
|
||||
source: PROVIDER,
|
||||
tenantId: source.tenantId,
|
||||
const identity = makeSyncIdentity(source, discovered);
|
||||
const sourceMetadata = {
|
||||
provider: identity.provider,
|
||||
sourceId: identity.sourceId,
|
||||
upstreamId: identity.upstreamId,
|
||||
owner: identity.owner,
|
||||
repo: identity.repo,
|
||||
ref: identity.ref,
|
||||
skillPath: identity.skillPath,
|
||||
commitSha,
|
||||
skillBlobSha: identity.skillBlobSha,
|
||||
syncedAt: serializeDate(syncedAt),
|
||||
syncStatus: identity.syncStatus,
|
||||
syncFingerprint: makeSyncFingerprint(identity, definition),
|
||||
};
|
||||
return { existing, update, createInput };
|
||||
const update: UpdateSkillInput = { ...definition, source: PROVIDER, sourceMetadata };
|
||||
return { existing, update, createInput: makeCreateInput(source, update, fallbackName) };
|
||||
}
|
||||
|
||||
async function commitRemoteSkill(
|
||||
|
|
@ -1609,14 +1764,23 @@ async function syncSource(params: {
|
|||
assertNotCancelled();
|
||||
try {
|
||||
assertGitHubSkillPackageManifest(discovered);
|
||||
const existing = await findExistingRemoteSkill(deps, source, discovered);
|
||||
assertNotCancelled();
|
||||
const unchanged = existing
|
||||
? await prepareUnchangedRemoteSkill({ deps, source, discovered, existing })
|
||||
: null;
|
||||
if (unchanged) {
|
||||
preparedSkills.push({ discovered, prepared: unchanged, unchanged: true });
|
||||
continue;
|
||||
}
|
||||
const skillMdPath = getSkillMdPath(discovered);
|
||||
const skillMdBuffer = await adapter.fetchFileContent(commit, discovered.skillMd);
|
||||
assertNotCancelled();
|
||||
assertGitHubBufferSize(skillMdBuffer, skillMdPath);
|
||||
const prepared = await prepareRemoteSkill({
|
||||
deps,
|
||||
source,
|
||||
discovered,
|
||||
existing,
|
||||
skillMdContent: skillMdBuffer.toString('utf-8'),
|
||||
commitSha: commit.id,
|
||||
syncedAt,
|
||||
|
|
@ -1706,10 +1870,45 @@ async function syncSource(params: {
|
|||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* A current mirror keeps every side effect a no-op file sync and skipped
|
||||
* update would have had: the public viewer grant and the cleanup of a
|
||||
* stale mirror holding its name. It is still counted as synced.
|
||||
*/
|
||||
const syncUnchangedSkill = async (
|
||||
discovered: DiscoveredSkill,
|
||||
prepared: PreparedExistingRemoteSkill,
|
||||
): Promise<void> => {
|
||||
await ensurePublicViewer(deps, prepared.existing._id);
|
||||
if (canReconcileStaleSkills) {
|
||||
const staleConflictCleanup = await deleteNameConflictingStaleSkill({
|
||||
deps,
|
||||
source,
|
||||
prepared,
|
||||
existingSyncedSkills: await getExistingSyncedSkills(),
|
||||
discoveredUpstreamIds,
|
||||
assertNotCancelled,
|
||||
});
|
||||
existingSyncedSkills = staleConflictCleanup.remainingSkills;
|
||||
counts.deletedSkillCount += staleConflictCleanup.deletedSkillCount;
|
||||
counts.deletedFileCount += staleConflictCleanup.deletedFileCount;
|
||||
if (staleConflictCleanup.deletedSkill) {
|
||||
await cleanupDeletedSyncedSkillFiles(deps, staleConflictCleanup.deletedSkill);
|
||||
}
|
||||
}
|
||||
counts.syncedSkillCount++;
|
||||
recordUnsupportedFiles(discovered);
|
||||
};
|
||||
|
||||
const syncPreparedSkill = async ({
|
||||
discovered,
|
||||
prepared,
|
||||
unchanged,
|
||||
}: PreparedDiscoveredSkill): Promise<void> => {
|
||||
if (unchanged && prepared.existing) {
|
||||
await syncUnchangedSkill(discovered, { ...prepared, existing: prepared.existing });
|
||||
return;
|
||||
}
|
||||
if (!prepared.existing && !canReconcileStaleSkills) {
|
||||
const ambiguousMovedMirror = findMovedSourceSkill({
|
||||
source,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue