mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-07-11 08:43:48 +00:00
fix: Address GitHub skill sync CI
This commit is contained in:
parent
fecdb793da
commit
5ba7e81739
9 changed files with 84 additions and 40 deletions
|
|
@ -3,10 +3,7 @@ const { createAdminSkillsSyncHandlers } = require('@librechat/api');
|
|||
const { SystemCapabilities } = require('@librechat/data-schemas');
|
||||
const { requireCapability } = require('~/server/middleware/roles/capabilities');
|
||||
const { requireJwtAuth } = require('~/server/middleware');
|
||||
const {
|
||||
upsertSkillSyncCredential,
|
||||
deleteSkillSyncCredential,
|
||||
} = require('~/models');
|
||||
const { upsertSkillSyncCredential, deleteSkillSyncCredential } = require('~/models');
|
||||
const { getGitHubSkillSyncRunner } = require('~/server/services/Skills/sync');
|
||||
|
||||
const router = express.Router();
|
||||
|
|
|
|||
|
|
@ -23,9 +23,7 @@ type AdminSkillsRequest = Request & {
|
|||
|
||||
export type AdminSkillSyncDeps = {
|
||||
runner: GitHubSkillSyncRunner;
|
||||
upsertCredential: (
|
||||
input: UpsertSkillSyncCredentialInput,
|
||||
) => Promise<SkillSyncCredentialSummary>;
|
||||
upsertCredential: (input: UpsertSkillSyncCredentialInput) => Promise<SkillSyncCredentialSummary>;
|
||||
deleteCredential: (
|
||||
provider: SkillSyncProvider,
|
||||
credentialKey: string,
|
||||
|
|
|
|||
|
|
@ -24,10 +24,7 @@ function extractFrontmatterBlock(raw: string): string | null {
|
|||
return normalized.slice(4, closingIdx);
|
||||
}
|
||||
|
||||
function getCaseInsensitive(
|
||||
frontmatter: Record<string, unknown>,
|
||||
key: string,
|
||||
): unknown {
|
||||
function getCaseInsensitive(frontmatter: Record<string, unknown>, key: string): unknown {
|
||||
const entry = Object.entries(frontmatter).find(([candidate]) => candidate.toLowerCase() === key);
|
||||
return entry?.[1];
|
||||
}
|
||||
|
|
@ -68,12 +65,12 @@ export function parseSkillMarkdown(raw: string): ParsedSkillMarkdown {
|
|||
const whenToUseValue = getCaseInsensitive(frontmatter, 'when-to-use');
|
||||
const alwaysApplyValue = getCaseInsensitive(frontmatter, 'always-apply');
|
||||
const name = typeof nameValue === 'string' ? nameValue : '';
|
||||
const description =
|
||||
typeof descriptionValue === 'string'
|
||||
? descriptionValue
|
||||
: typeof whenToUseValue === 'string'
|
||||
? whenToUseValue
|
||||
: '';
|
||||
let description = '';
|
||||
if (typeof descriptionValue === 'string') {
|
||||
description = descriptionValue;
|
||||
} else if (typeof whenToUseValue === 'string') {
|
||||
description = whenToUseValue;
|
||||
}
|
||||
let alwaysApply: boolean | undefined;
|
||||
const invalidBooleans: string[] = [];
|
||||
if (alwaysApplyValue !== undefined) {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ function response(body: unknown, status = 200): Response {
|
|||
get: () => null,
|
||||
},
|
||||
json: async () => body,
|
||||
} as Response;
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
function blob(content: string) {
|
||||
|
|
@ -172,7 +172,7 @@ function createDeps(
|
|||
return response(blob('echo ok'));
|
||||
}
|
||||
return response({ message: 'not found' }, 404);
|
||||
}),
|
||||
}) as unknown as typeof fetch,
|
||||
...overrides,
|
||||
};
|
||||
return deps;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,7 @@
|
|||
import crypto from 'crypto';
|
||||
import path from 'path';
|
||||
import { Types } from 'mongoose';
|
||||
import {
|
||||
ResourceType,
|
||||
PrincipalType,
|
||||
AccessRoleIds,
|
||||
} from 'librechat-data-provider';
|
||||
import { ResourceType, PrincipalType, AccessRoleIds } from 'librechat-data-provider';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import type { SkillSyncConfig, SkillSyncGitHubSourceConfig } from 'librechat-data-provider';
|
||||
import type {
|
||||
|
|
@ -87,7 +83,10 @@ type SaveBufferResult = {
|
|||
|
||||
export type GitHubSkillSyncDeps = {
|
||||
getConfig: () => SkillSyncConfig | undefined;
|
||||
getCredentialToken: (provider: SkillSyncProvider, credentialKey: string) => Promise<string | null>;
|
||||
getCredentialToken: (
|
||||
provider: SkillSyncProvider,
|
||||
credentialKey: string,
|
||||
) => Promise<string | null>;
|
||||
getCredentialSummary: (
|
||||
provider: SkillSyncProvider,
|
||||
credentialKey: string,
|
||||
|
|
@ -266,7 +265,10 @@ function sanitizeError(error: unknown): { code: string; message: string } {
|
|||
return { code: error.code, message: error.message };
|
||||
}
|
||||
if (error instanceof Error) {
|
||||
return { code: 'SYNC_FAILED', message: error.message.replace(/Bearer\s+\S+/gi, 'Bearer [redacted]') };
|
||||
return {
|
||||
code: 'SYNC_FAILED',
|
||||
message: error.message.replace(/Bearer\s+\S+/gi, 'Bearer [redacted]'),
|
||||
};
|
||||
}
|
||||
return { code: 'SYNC_FAILED', message: 'Unknown skill sync failure' };
|
||||
}
|
||||
|
|
@ -303,7 +305,10 @@ async function githubJson<T>(params: {
|
|||
if (response.status === 404) {
|
||||
throw new SkillSyncError('GITHUB_NOT_FOUND', 'GitHub repository, ref, or path was not found');
|
||||
}
|
||||
throw new SkillSyncError('GITHUB_REQUEST_FAILED', `GitHub request failed with HTTP ${response.status}`);
|
||||
throw new SkillSyncError(
|
||||
'GITHUB_REQUEST_FAILED',
|
||||
`GitHub request failed with HTTP ${response.status}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchCommit(params: {
|
||||
|
|
@ -352,7 +357,10 @@ async function fetchBlob(params: {
|
|||
pathname: `/repos/${owner}/${repo}/git/blobs/${sha}`,
|
||||
});
|
||||
if (blob.encoding !== 'base64') {
|
||||
throw new SkillSyncError('GITHUB_UNSUPPORTED_BLOB', `Unsupported GitHub blob encoding "${blob.encoding}"`);
|
||||
throw new SkillSyncError(
|
||||
'GITHUB_UNSUPPORTED_BLOB',
|
||||
`Unsupported GitHub blob encoding "${blob.encoding}"`,
|
||||
);
|
||||
}
|
||||
return Buffer.from(blob.content.replace(/\s/g, ''), 'base64');
|
||||
}
|
||||
|
|
@ -521,7 +529,10 @@ async function upsertRemoteSkill(params: {
|
|||
if (result.status === 'conflict') {
|
||||
throw new SkillSyncError('SKILL_CONFLICT', `Skill "${existing.name}" changed during sync`);
|
||||
}
|
||||
throw new SkillSyncError('SKILL_NOT_FOUND', `Previously synced skill "${existing.name}" was removed`);
|
||||
throw new SkillSyncError(
|
||||
'SKILL_NOT_FOUND',
|
||||
`Previously synced skill "${existing.name}" was removed`,
|
||||
);
|
||||
}
|
||||
const createInput: CreateSkillInput = {
|
||||
...(update as Omit<UpdateSkillInput, 'source'>),
|
||||
|
|
@ -628,7 +639,10 @@ async function syncSkillFiles(params: {
|
|||
tenantId: skill.tenantId,
|
||||
})
|
||||
.catch((cleanupError) =>
|
||||
logger.error('[GitHubSkillSync] Failed to clean up orphaned synced file:', cleanupError),
|
||||
logger.error(
|
||||
'[GitHubSkillSync] Failed to clean up orphaned synced file:',
|
||||
cleanupError,
|
||||
),
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
|
|
@ -683,7 +697,10 @@ async function syncSource(params: {
|
|||
try {
|
||||
const token = await deps.getCredentialToken(PROVIDER, source.credentialKey);
|
||||
if (!token) {
|
||||
throw new SkillSyncError('MISSING_CREDENTIAL', `Missing GitHub credential "${source.credentialKey}"`);
|
||||
throw new SkillSyncError(
|
||||
'MISSING_CREDENTIAL',
|
||||
`Missing GitHub credential "${source.credentialKey}"`,
|
||||
);
|
||||
}
|
||||
const commit = await fetchCommit({ fetchFn, token, source });
|
||||
const tree = await fetchTree({ fetchFn, token, source, treeSha: commit.commit.tree.sha });
|
||||
|
|
@ -702,7 +719,12 @@ async function syncSource(params: {
|
|||
const syncedAt = new Date();
|
||||
|
||||
for (const discovered of discoveredSkills) {
|
||||
const skillMdBuffer = await fetchBlob({ fetchFn, token, source, sha: discovered.skillMd.sha });
|
||||
const skillMdBuffer = await fetchBlob({
|
||||
fetchFn,
|
||||
token,
|
||||
source,
|
||||
sha: discovered.skillMd.sha,
|
||||
});
|
||||
const skill = await upsertRemoteSkill({
|
||||
deps,
|
||||
source,
|
||||
|
|
@ -726,7 +748,10 @@ async function syncSource(params: {
|
|||
counts.deletedFileCount += fileCounts.deletedFileCount;
|
||||
}
|
||||
|
||||
const existingSyncedSkills = await deps.listSkillsBySource({ source: PROVIDER, sourceId: source.id });
|
||||
const existingSyncedSkills = await deps.listSkillsBySource({
|
||||
source: PROVIDER,
|
||||
sourceId: source.id,
|
||||
});
|
||||
for (const skill of existingSyncedSkills) {
|
||||
const upstreamId =
|
||||
skill.sourceMetadata && typeof skill.sourceMetadata.upstreamId === 'string'
|
||||
|
|
@ -790,7 +815,9 @@ export function createGitHubSkillSyncRunner(deps: GitHubSkillSyncDeps) {
|
|||
deps.listCredentials(PROVIDER),
|
||||
]);
|
||||
const statusBySourceId = new Map(storedStatuses.map((status) => [status.sourceId, status]));
|
||||
const credentialByKey = new Map(credentials.map((credential) => [credential.credentialKey, credential]));
|
||||
const credentialByKey = new Map(
|
||||
credentials.map((credential) => [credential.credentialKey, credential]),
|
||||
);
|
||||
const sources = github.sources.map((source) => {
|
||||
const stored = statusBySourceId.get(source.id);
|
||||
const credential = credentialByKey.get(source.credentialKey);
|
||||
|
|
@ -840,7 +867,11 @@ export function createGitHubSkillSyncRunner(deps: GitHubSkillSyncDeps) {
|
|||
});
|
||||
if (!acquired) {
|
||||
const statuses = await deps.listStatuses(PROVIDER);
|
||||
return { status: 'skipped', message: 'GitHub skill sync is already running', sources: statuses };
|
||||
return {
|
||||
status: 'skipped',
|
||||
message: 'GitHub skill sync is already running',
|
||||
sources: statuses,
|
||||
};
|
||||
}
|
||||
try {
|
||||
const sources: ISkillSyncStatus[] = [];
|
||||
|
|
|
|||
|
|
@ -241,7 +241,8 @@ const skillSyncIdentifierSchema = z
|
|||
.min(1)
|
||||
.max(64)
|
||||
.regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, {
|
||||
message: 'must start with a letter or digit and contain only letters, digits, underscores, or hyphens',
|
||||
message:
|
||||
'must start with a letter or digit and contain only letters, digits, underscores, or hyphens',
|
||||
});
|
||||
|
||||
const skillSyncGitHubOwnerSchema = z
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import {
|
||||
EModelEndpoint,
|
||||
getConfigDefaults,
|
||||
skillSyncConfigSchema,
|
||||
summarizationConfigSchema,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TCustomConfig, FileSources, DeepPartial } from 'librechat-data-provider';
|
||||
|
|
@ -51,6 +52,21 @@ export function loadSummarizationConfig(
|
|||
};
|
||||
}
|
||||
|
||||
export function loadSkillSyncConfig(config: DeepPartial<TCustomConfig>): AppConfig['skillSync'] {
|
||||
const raw = config.skillSync;
|
||||
if (!raw || typeof raw !== 'object') {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = skillSyncConfigSchema.safeParse(raw);
|
||||
if (!parsed.success) {
|
||||
logger.warn('[AppService] Invalid skill sync config', parsed.error.flatten());
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
export type Paths = {
|
||||
root: string;
|
||||
uploads: string;
|
||||
|
|
@ -83,7 +99,7 @@ export const AppService = async (params?: {
|
|||
const webSearch = loadWebSearchConfig(config.webSearch);
|
||||
const memory = loadMemoryConfig(config.memory);
|
||||
const summarization = loadSummarizationConfig(config);
|
||||
const skillSync = config.skillSync;
|
||||
const skillSync = loadSkillSyncConfig(config);
|
||||
const filteredTools = config.filteredTools;
|
||||
const includedTools = config.includedTools;
|
||||
const fileStrategy = (config.fileStrategy ?? configDefaults.fileStrategy) as
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import mongoose from 'mongoose';
|
|||
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||
import { createModels } from '../models';
|
||||
import { createSkillSyncMethods } from './skillSync';
|
||||
import type { ISkillSyncCredential } from '~/types/skillSync';
|
||||
import { encryptV2, decryptV2 } from '~/crypto';
|
||||
|
||||
jest.mock('~/crypto', () => ({
|
||||
|
|
@ -46,12 +47,12 @@ describe('createSkillSyncMethods', () => {
|
|||
expect(JSON.stringify(summary)).not.toContain('ghp_secret');
|
||||
expect(JSON.stringify(summary)).not.toContain('encrypted:ghp_secret');
|
||||
|
||||
const raw = await mongoose.models.SkillSyncCredential.findOne({
|
||||
const raw = (await mongoose.models.SkillSyncCredential.findOne({
|
||||
provider: 'github',
|
||||
credentialKey: 'github-skills-prod',
|
||||
})
|
||||
.select('+encryptedToken +tokenHash')
|
||||
.lean();
|
||||
.lean()) as ISkillSyncCredential | null;
|
||||
if (!raw) {
|
||||
throw new Error('Expected stored skill sync credential');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,7 +109,10 @@ export function createSkillSyncMethods(mongoose: typeof import('mongoose')) {
|
|||
provider: SkillSyncProvider,
|
||||
): Promise<SkillSyncCredentialSummary[]> {
|
||||
const Credential = mongoose.models.SkillSyncCredential as Model<ISkillSyncCredentialDocument>;
|
||||
const rows = await Credential.find({ provider }).select('+tokenHash').sort({ credentialKey: 1 }).lean();
|
||||
const rows = await Credential.find({ provider })
|
||||
.select('+tokenHash')
|
||||
.sort({ credentialKey: 1 })
|
||||
.lean();
|
||||
return rows.map((row) => summarizeCredential(row as ISkillSyncCredential));
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue