diff --git a/api/server/experimental.js b/api/server/experimental.js index 48666a2f7d..c5584c734f 100644 --- a/api/server/experimental.js +++ b/api/server/experimental.js @@ -25,8 +25,10 @@ const { } = require('@librechat/api'); const { connectDb, indexSync } = require('~/db'); const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager'); +const { capabilityContextMiddleware } = require('./middleware/roles/capabilities'); const createValidateImageRequest = require('./middleware/validateImageRequest'); const { startExpiredFileSweep } = require('./services/Files/process'); +const { initializeGitHubSkillSync } = require('./services/Skills/sync'); const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies'); const { updateInterfacePermissions: updateInterfacePerms } = require('@librechat/api'); const { @@ -296,6 +298,7 @@ if (cluster.isMaster) { /** Initialize app configuration */ const appConfig = await getAppConfig(); initializeFileStorage(appConfig); + initializeGitHubSkillSync(appConfig); expiredFileSweepOptions = { appConfig, loadAppConfig: getAppConfig }; startExpiredFileSweepOnce(); await performStartupChecks(appConfig); @@ -390,10 +393,13 @@ if (cluster.isMaster) { await configureSocialLogins(app); } + app.use(capabilityContextMiddleware); + /** Routes */ app.use('/oauth', routes.oauth); app.use('/api/auth', routes.auth); app.use('/api/admin', routes.adminAuth); + app.use('/api/admin/skills', routes.adminSkills); app.use('/api/actions', routes.actions); app.use('/api/keys', routes.keys); app.use('/api/api-keys', routes.apiKeys); diff --git a/api/server/index.js b/api/server/index.js index 19ef7d533c..a1161fd221 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -39,6 +39,7 @@ const initializeOAuthReconnectManager = require('./services/initializeOAuthRecon const { capabilityContextMiddleware } = require('./middleware/roles/capabilities'); const createValidateImageRequest = require('./middleware/validateImageRequest'); const { startExpiredFileSweep } = require('./services/Files/process'); +const { initializeGitHubSkillSync } = require('./services/Skills/sync'); const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies'); const { checkMigrations } = require('./services/start/migration'); const optionalJwtAuth = require('./middleware/optionalJwtAuth'); @@ -120,6 +121,7 @@ const startServer = async () => { const appConfig = await getAppConfig({ baseOnly: true }); initializeFileStorage(appConfig); await initializeDeploymentSkills({ projectRoot: path.resolve(__dirname, '../..') }); + initializeGitHubSkillSync(appConfig); startExpiredFileSweep({ appConfig, loadAppConfig: getAppConfig }); await runAsSystem(async () => { await performStartupChecks(appConfig); @@ -238,6 +240,7 @@ const startServer = async () => { app.use('/api/admin/grants', routes.adminGrants); app.use('/api/admin/groups', routes.adminGroups); app.use('/api/admin/roles', routes.adminRoles); + app.use('/api/admin/skills', routes.adminSkills); app.use('/api/admin/users', routes.adminUsers); app.use('/api/actions', routes.actions); app.use('/api/keys', routes.keys); diff --git a/api/server/routes/admin/skills.js b/api/server/routes/admin/skills.js new file mode 100644 index 0000000000..54e54004ff --- /dev/null +++ b/api/server/routes/admin/skills.js @@ -0,0 +1,50 @@ +const express = require('express'); +const { createAdminSkillsSyncAccess, createAdminSkillsSyncHandlers } = require('@librechat/api'); +const { SystemCapabilities } = require('@librechat/data-schemas'); +const { hasCapability, requireCapability } = require('~/server/middleware/roles/capabilities'); +const { requireJwtAuth } = require('~/server/middleware'); +const { upsertSkillSyncCredential, deleteSkillSyncCredential } = require('~/models'); +const { getGitHubSkillSyncRunnerForRequest } = require('~/server/services/Skills/sync'); +const { getAppConfig } = require('~/server/services/Config'); +const configMiddleware = require('~/server/middleware/config/app'); + +const router = express.Router(); +const requireAdminAccess = requireCapability(SystemCapabilities.ACCESS_ADMIN); + +const syncAccess = createAdminSkillsSyncAccess({ + getAppConfig, + hasCapability, +}); + +const handlers = createAdminSkillsSyncHandlers({ + getRunner: getGitHubSkillSyncRunnerForRequest, + upsertCredential: upsertSkillSyncCredential, + deleteCredential: deleteSkillSyncCredential, +}); + +router.use( + requireJwtAuth, + requireAdminAccess, + configMiddleware, + syncAccess.attachBaseSkillSyncConfig, +); + +router.get( + '/sync/status', + syncAccess.requireReadSkills, + syncAccess.attachCredentialReadAccess, + handlers.getSyncStatus, +); +router.post('/sync/run', syncAccess.requireSyncRunCapability, handlers.runSync); +router.put( + '/sync/credentials/:credentialKey', + syncAccess.requirePlatformManageSkills, + handlers.setCredential, +); +router.delete( + '/sync/credentials/:credentialKey', + syncAccess.requirePlatformManageSkills, + handlers.deleteCredential, +); + +module.exports = router; diff --git a/api/server/routes/admin/skills.test.js b/api/server/routes/admin/skills.test.js new file mode 100644 index 0000000000..f452d6ea21 --- /dev/null +++ b/api/server/routes/admin/skills.test.js @@ -0,0 +1,119 @@ +const express = require('express'); +const request = require('supertest'); + +const mockRequireJwtAuth = jest.fn((req, res, next) => { + req.user = { id: 'user-1', role: 'ADMIN', tenantId: 'tenant-a' }; + next(); +}); +const mockCapabilityMiddleware = jest.fn((req, res, next) => next()); +const mockRequireCapability = jest.fn(() => mockCapabilityMiddleware); +const mockHasCapability = jest.fn().mockResolvedValue(true); +const mockConfigMiddleware = jest.fn((req, res, next) => { + req.config = { skillSync: { github: { enabled: false, sources: [] } } }; + next(); +}); +const mockGetAppConfig = jest.fn(); +const mockGetGitHubSkillSyncRunnerForRequest = jest.fn(); +const mockHandlers = { + getSyncStatus: jest.fn((req, res) => res.status(200).json({ ok: true })), + runSync: jest.fn((req, res) => res.status(200).json({ ok: true })), + setCredential: jest.fn((req, res) => res.status(200).json({ ok: true })), + deleteCredential: jest.fn((req, res) => res.status(200).json({ ok: true })), +}; +const mockSyncAccess = { + attachBaseSkillSyncConfig: jest.fn((req, res, next) => next()), + requireReadSkills: jest.fn((req, res, next) => next()), + attachCredentialReadAccess: jest.fn((req, res, next) => next()), + requireSyncRunCapability: jest.fn((req, res, next) => next()), + requirePlatformManageSkills: jest.fn((req, res, next) => next()), +}; + +jest.mock('@librechat/data-schemas', () => ({ + SystemCapabilities: { + ACCESS_ADMIN: 'access:admin', + }, +})); + +jest.mock('@librechat/api', () => ({ + createAdminSkillsSyncAccess: jest.fn(() => mockSyncAccess), + createAdminSkillsSyncHandlers: jest.fn(() => mockHandlers), +})); + +jest.mock('~/server/middleware/roles/capabilities', () => ({ + hasCapability: mockHasCapability, + requireCapability: mockRequireCapability, +})); + +jest.mock('~/server/middleware', () => ({ + requireJwtAuth: mockRequireJwtAuth, +})); + +jest.mock('~/server/middleware/config/app', () => mockConfigMiddleware); + +jest.mock('~/server/services/Config', () => ({ + getAppConfig: mockGetAppConfig, +})); + +jest.mock('~/models', () => ({ + upsertSkillSyncCredential: jest.fn(), + deleteSkillSyncCredential: jest.fn(), +})); + +jest.mock('~/server/services/Skills/sync', () => ({ + getGitHubSkillSyncRunnerForRequest: mockGetGitHubSkillSyncRunnerForRequest, +})); + +describe('admin skills sync routes', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + function createApp() { + delete require.cache[require.resolve('./skills')]; + const router = require('./skills'); + const app = express(); + app.use(express.json()); + app.use('/api/admin/skills', router); + return app; + } + + it('delegates skill sync access policy to the API package', async () => { + const app = createApp(); + + await request(app).get('/api/admin/skills/sync/status').expect(200); + + const { + createAdminSkillsSyncAccess, + createAdminSkillsSyncHandlers, + } = require('@librechat/api'); + expect(mockRequireCapability).toHaveBeenCalledWith('access:admin'); + expect(createAdminSkillsSyncAccess).toHaveBeenCalledWith({ + getAppConfig: mockGetAppConfig, + hasCapability: mockHasCapability, + }); + expect(createAdminSkillsSyncHandlers).toHaveBeenCalledWith( + expect.objectContaining({ getRunner: mockGetGitHubSkillSyncRunnerForRequest }), + ); + expect(mockRequireJwtAuth).toHaveBeenCalled(); + expect(mockCapabilityMiddleware).toHaveBeenCalled(); + expect(mockConfigMiddleware).toHaveBeenCalled(); + expect(mockSyncAccess.attachBaseSkillSyncConfig).toHaveBeenCalled(); + expect(mockSyncAccess.requireReadSkills).toHaveBeenCalled(); + expect(mockSyncAccess.attachCredentialReadAccess).toHaveBeenCalled(); + expect(mockHandlers.getSyncStatus).toHaveBeenCalled(); + }); + + it('mounts package access middlewares before each sync endpoint handler', async () => { + const app = createApp(); + + await request(app).post('/api/admin/skills/sync/run').expect(200); + await request(app).put('/api/admin/skills/sync/credentials/default').send({}).expect(200); + await request(app).delete('/api/admin/skills/sync/credentials/default').expect(200); + + expect(mockSyncAccess.requireSyncRunCapability).toHaveBeenCalled(); + expect(mockHandlers.runSync).toHaveBeenCalled(); + expect(mockSyncAccess.requirePlatformManageSkills).toHaveBeenCalledTimes(2); + expect(mockHandlers.setCredential).toHaveBeenCalled(); + expect(mockHandlers.deleteCredential).toHaveBeenCalled(); + }); +}); diff --git a/api/server/routes/index.js b/api/server/routes/index.js index 3322dd57e3..59955e937e 100644 --- a/api/server/routes/index.js +++ b/api/server/routes/index.js @@ -6,6 +6,7 @@ const adminConfig = require('./admin/config'); const adminGrants = require('./admin/grants'); const adminGroups = require('./admin/groups'); const adminRoles = require('./admin/roles'); +const adminSkills = require('./admin/skills'); const adminUsers = require('./admin/users'); const endpoints = require('./endpoints'); const staticRoute = require('./static'); @@ -44,6 +45,7 @@ module.exports = { adminGrants, adminGroups, adminRoles, + adminSkills, adminUsers, keys, apiKeys, diff --git a/api/server/routes/skills.js b/api/server/routes/skills.js index eb756a432f..99339d2a29 100644 --- a/api/server/routes/skills.js +++ b/api/server/routes/skills.js @@ -37,6 +37,7 @@ const { } = require('~/server/services/PermissionService'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { createFileLimiters } = require('~/server/middleware/limiters/uploadLimiters'); +const { maybeRunGitHubSkillSyncForRequest } = require('~/server/services/Skills/sync'); const configMiddleware = require('~/server/middleware/config/app'); const { getFileStrategy } = require('~/server/utils/getFileStrategy'); const { @@ -285,6 +286,14 @@ async function uploadFileHandler(req, res) { // --------------------------------------------------------------------------- // Routes // --------------------------------------------------------------------------- +async function maybeStartRequestSkillSync(req, _res, next) { + try { + await maybeRunGitHubSkillSyncForRequest(req); + } catch (error) { + logger.error('[GET /skills] Failed to start request-scoped skill sync:', error); + } + next(); +} // Import: accepts .md / .zip / .skill via multipart router.post( @@ -297,7 +306,7 @@ router.post( importHandler, ); -router.get('/', handlers.list); +router.get('/', maybeStartRequestSkillSync, handlers.list); router.post('/', checkSkillCreate, handlers.create); router.get( diff --git a/api/server/routes/skills.test.js b/api/server/routes/skills.test.js index af99e4bc5e..c48c0ff70b 100644 --- a/api/server/routes/skills.test.js +++ b/api/server/routes/skills.test.js @@ -33,6 +33,7 @@ const { } = require('librechat-data-provider'); let mockFileConfig; +const mockMaybeRunGitHubSkillSyncForRequest = jest.fn(async () => false); jest.mock('~/server/services/Config', () => ({ getCachedTools: jest.fn().mockResolvedValue({}), @@ -68,6 +69,10 @@ jest.mock('~/server/utils/getFileStrategy', () => ({ getFileStrategy: jest.fn().mockReturnValue('local'), })); +jest.mock('~/server/services/Skills/sync', () => ({ + maybeRunGitHubSkillSyncForRequest: mockMaybeRunGitHubSkillSyncForRequest, +})); + jest.mock('~/models', () => { const mongoose = require('mongoose'); const { createMethods } = require('@librechat/data-schemas'); @@ -152,6 +157,7 @@ afterEach(async () => { await AclEntry.deleteMany({}); currentTestUser = testUsers.owner; mockFileConfig = undefined; + mockMaybeRunGitHubSkillSyncForRequest.mockClear(); }); afterAll(async () => { @@ -409,6 +415,12 @@ describe('Skill routes', () => { setTestUser(testUsers.owner); const res = await request(app).get('/api/skills'); expect(res.status).toBe(200); + expect(mockMaybeRunGitHubSkillSyncForRequest).toHaveBeenCalledWith( + expect.objectContaining({ + config: expect.objectContaining({ fileStrategy: 'local' }), + user: expect.objectContaining({ id: testUsers.owner._id.toString() }), + }), + ); expect(res.body.skills.length).toBe(1); expect(res.body.skills[0].name).toBe('mine-skill'); }); diff --git a/api/server/services/Skills/sync.js b/api/server/services/Skills/sync.js new file mode 100644 index 0000000000..f19005639f --- /dev/null +++ b/api/server/services/Skills/sync.js @@ -0,0 +1,215 @@ +const { FileContext } = require('librechat-data-provider'); +const { + getStorageMetadata, + createGitHubSkillSyncRunner, + createSkillSyncTriggerOrchestrator, + startGitHubSkillSyncScheduler, +} = require('@librechat/api'); +const { logger, runAsSystem } = require('@librechat/data-schemas'); +const db = require('~/models'); +const { getAppConfig } = require('~/server/services/Config'); +const { getStrategyFunctions } = require('~/server/services/Files/strategies'); +const { getFileStrategy } = require('~/server/utils/getFileStrategy'); + +const SYSTEM_USER_ID = '000000000000000000000000'; + +let appConfigRef; +let runner; +let scheduler; + +async function loadCurrentAppConfig() { + try { + const appConfig = await getAppConfig({ baseOnly: true }); + appConfigRef = appConfig; + return appConfig; + } catch (error) { + if (appConfigRef) { + return appConfigRef; + } + throw error; + } +} + +async function getSyncConfig(loadAppConfig = loadCurrentAppConfig) { + const appConfig = await loadAppConfig(); + return appConfig?.skillSync; +} + +async function resolveSkillStorage({ isImage = false, loadAppConfig = loadCurrentAppConfig } = {}) { + const appConfig = await loadAppConfig(); + const source = getFileStrategy(appConfig, { context: FileContext.skill_file, isImage }); + const strategy = getStrategyFunctions(source); + if (!strategy.saveBuffer) { + throw new Error(`Storage backend "${source}" does not support file writes`); + } + return { source, saveBuffer: strategy.saveBuffer }; +} + +async function getSyntheticReq({ userId = SYSTEM_USER_ID, tenantId, loadAppConfig } = {}) { + const appConfig = await (loadAppConfig ?? loadCurrentAppConfig)(); + return { + config: appConfig, + user: { + id: userId, + _id: userId, + tenantId, + }, + }; +} + +function withBaseSkillSyncConfig(req, baseConfig) { + if (!req?.config || req.config.config?.skillSync !== undefined) { + return req; + } + return { + ...req, + config: { + ...req.config, + config: { + ...(req.config.config ?? {}), + skillSync: baseConfig?.skillSync, + }, + }, + }; +} + +function createRunner({ getConfig, loadAppConfig, allowServerCredentials = true } = {}) { + const resolveAppConfig = loadAppConfig ?? loadCurrentAppConfig; + const resolveConfig = getConfig ?? (() => getSyncConfig(resolveAppConfig)); + const createdRunner = createGitHubSkillSyncRunner({ + getConfig: resolveConfig, + getCredentialToken: db.getSkillSyncCredentialToken, + getCredentialSummary: db.getSkillSyncCredentialSummary, + listCredentials: db.listSkillSyncCredentials, + listStatuses: db.listSkillSyncStatuses, + upsertStatus: db.upsertSkillSyncStatus, + tryAcquireLock: db.tryAcquireSkillSyncLock, + refreshLock: db.refreshSkillSyncLock, + releaseLock: db.releaseSkillSyncLock, + createSkill: db.createSkill, + updateSkill: db.updateSkill, + getSkillById: db.getSkillById, + findSkillBySourceIdentity: db.findSkillBySourceIdentity, + listSkillsBySource: db.listSkillsBySource, + listSkillFiles: db.listSkillFiles, + getSkillFileByPath: db.getSkillFileByPath, + upsertSkillFile: db.upsertSkillFile, + deleteSkillFile: db.deleteSkillFile, + deleteSkill: db.deleteSkill, + grantPermission: async ({ + principalType, + principalId, + resourceType, + resourceId, + accessRoleId, + grantedBy, + }) => { + // Default access roles are seeded globally (no tenantId) under runAsSystem, + // but the runner may execute inside a source's tenant context. Resolve the + // role outside tenant isolation so the global role matches, then write the + // ACL entry in the active (tenant) context so tenant users can see it. + const role = await runAsSystem(() => db.findRoleByIdentifier(accessRoleId)); + if (!role) { + throw new Error(`Role ${accessRoleId} not found`); + } + if (role.resourceType !== resourceType) { + throw new Error( + `Role ${accessRoleId} is for ${role.resourceType} resources, not ${resourceType}`, + ); + } + return db.grantPermission( + principalType, + principalId, + resourceType, + resourceId, + role.permBits, + grantedBy, + undefined, + role._id, + ); + }, + saveBuffer: async ({ userId, buffer, fileName, basePath, isImage, tenantId }) => { + const storage = await resolveSkillStorage({ isImage, loadAppConfig: resolveAppConfig }); + const filepath = await storage.saveBuffer({ + userId: userId ?? SYSTEM_USER_ID, + buffer, + fileName, + basePath, + tenantId, + }); + return { + filepath, + source: storage.source, + ...getStorageMetadata({ filepath, source: storage.source }), + }; + }, + deleteFile: async (file) => { + const strategy = getStrategyFunctions(file.source); + if (!strategy.deleteFile) { + return; + } + await strategy.deleteFile( + await getSyntheticReq({ + userId: file.user?.toString?.() ?? file.user ?? SYSTEM_USER_ID, + tenantId: file.tenantId, + loadAppConfig: resolveAppConfig, + }), + file, + ); + }, + allowServerCredentials, + }); + return { + getStatus: createdRunner.getStatus, + runOnce: createdRunner.runOnce, + }; +} + +const triggerOrchestrator = createSkillSyncTriggerOrchestrator({ + createRunner, + logger, +}); + +function getGitHubSkillSyncRunnerForRequest(req) { + return triggerOrchestrator.getRunnerForAdminRequest(withBaseSkillSyncConfig(req, appConfigRef)); +} + +async function maybeRunGitHubSkillSyncForRequest(req) { + const baseConfig = await loadCurrentAppConfig(); + return triggerOrchestrator.maybeRunForRequest({ + ...withBaseSkillSyncConfig(req, baseConfig), + skillSyncAllowServerCredentials: false, + }); +} + +function initializeGitHubSkillSync(appConfig) { + appConfigRef = appConfig; + runner = createRunner(); + scheduler = startGitHubSkillSyncScheduler({ + getConfig: getSyncConfig, + runner, + }); + return { runner, scheduler }; +} + +function getGitHubSkillSyncRunner() { + if (!runner) { + runner = createRunner(); + } + return runner; +} + +function stopGitHubSkillSyncScheduler() { + if (scheduler) { + scheduler.stop(); + scheduler = undefined; + } +} + +module.exports = { + initializeGitHubSkillSync, + getGitHubSkillSyncRunner, + getGitHubSkillSyncRunnerForRequest, + maybeRunGitHubSkillSyncForRequest, + stopGitHubSkillSyncScheduler, +}; diff --git a/api/server/services/Skills/sync.test.js b/api/server/services/Skills/sync.test.js new file mode 100644 index 0000000000..069ffc6db2 --- /dev/null +++ b/api/server/services/Skills/sync.test.js @@ -0,0 +1,583 @@ +const mockGetAppConfig = jest.fn(); +const mockGetStrategyFunctions = jest.fn(); +const mockGetFileStrategy = jest.fn(); +const mockFindRoleByIdentifier = jest.fn(); +const mockGrantPermission = jest.fn(); +let mockRunnerDeps; +let mockRunnerStatus; +const mockCreatedRunners = []; + +jest.mock('~/server/services/Config', () => ({ + getAppConfig: mockGetAppConfig, +})); + +jest.mock('@librechat/api', () => { + const actualApi = jest.requireActual('@librechat/api'); + return { + createSkillSyncTriggerOrchestrator: actualApi.createSkillSyncTriggerOrchestrator, + createGitHubSkillSyncRunner: jest.fn((deps) => { + mockRunnerDeps = deps; + const runner = { + getStatus: jest.fn(async () => { + if (mockRunnerStatus) { + return mockRunnerStatus; + } + const config = await deps.getConfig(); + const github = config?.github ?? {}; + return { + enabled: github.enabled ?? false, + intervalMinutes: github.intervalMinutes ?? 60, + runOnStartup: github.runOnStartup ?? false, + sources: (github.sources ?? []).map((source) => ({ + provider: 'github', + sourceId: source.id, + status: 'idle', + credentialPresent: + deps.allowServerCredentials !== false && + Boolean(source.credentialKey || source.token), + owner: source.owner, + repo: source.repo, + ref: source.ref, + paths: source.paths, + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + })), + credentials: [], + }; + }), + runOnce: jest.fn(async () => deps.getConfig()), + }; + mockCreatedRunners.push({ deps, runner }); + return runner; + }), + getStorageMetadata: jest.fn(() => ({})), + startGitHubSkillSyncScheduler: jest.fn(() => ({ stop: jest.fn() })), + }; +}); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { + error: jest.fn(), + warn: jest.fn(), + }, + runAsSystem: jest.fn((fn) => fn()), +})); + +jest.mock('~/models', () => ({ + findRoleByIdentifier: mockFindRoleByIdentifier, + grantPermission: mockGrantPermission, + getSkillSyncCredentialToken: jest.fn(), + getSkillSyncCredentialSummary: jest.fn(), + listSkillSyncCredentials: jest.fn(async () => []), + listSkillSyncStatuses: jest.fn(async () => []), + upsertSkillSyncStatus: jest.fn(), + tryAcquireSkillSyncLock: jest.fn(), + refreshSkillSyncLock: jest.fn(), + releaseSkillSyncLock: jest.fn(), + createSkill: jest.fn(), + updateSkill: jest.fn(), + getSkillById: jest.fn(), + findSkillBySourceIdentity: jest.fn(), + listSkillsBySource: jest.fn(), + listSkillFiles: jest.fn(), + getSkillFileByPath: jest.fn(), + upsertSkillFile: jest.fn(), + deleteSkillFile: jest.fn(), + deleteSkill: jest.fn(), +})); +jest.mock('~/server/services/Files/strategies', () => ({ + getStrategyFunctions: mockGetStrategyFunctions, +})); +jest.mock('~/server/utils/getFileStrategy', () => ({ getFileStrategy: mockGetFileStrategy })); + +describe('GitHub skill sync service', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockGetAppConfig.mockReset(); + mockGetStrategyFunctions.mockReset(); + mockGetFileStrategy.mockReset(); + mockFindRoleByIdentifier.mockReset(); + mockGrantPermission.mockReset(); + mockRunnerDeps = undefined; + mockRunnerStatus = undefined; + mockCreatedRunners.length = 0; + }); + + it('resolves sync config from fresh base app config for runner operations', async () => { + const startupSkillSync = { + github: { + enabled: false, + intervalMinutes: 60, + runOnStartup: false, + sources: [], + }, + }; + const freshSkillSync = { + github: { + enabled: true, + intervalMinutes: 5, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }; + mockGetAppConfig.mockResolvedValue({ skillSync: freshSkillSync }); + + const service = require('./sync'); + const { runner } = service.initializeGitHubSkillSync({ skillSync: startupSkillSync }); + const result = await runner.runOnce(); + + expect(result).toBe(freshSkillSync); + expect(mockRunnerDeps.getConfig).toBeDefined(); + expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true }); + }); + + it('does not return raw unvalidated config.skillSync as sync config', async () => { + const rawSkillSync = { + github: { + enabled: true, + sources: 'not-an-array', + }, + }; + mockGetAppConfig.mockResolvedValue({ config: { skillSync: rawSkillSync } }); + + const service = require('./sync'); + const { runner } = service.initializeGitHubSkillSync({ config: { skillSync: rawSkillSync } }); + const result = await runner.runOnce(); + + expect(result).toBeUndefined(); + expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true }); + }); + + it('does not let user skill-list sync use server credentials from resolved config', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [ + { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + tenantId: 'other-tenant', + }, + ], + }, + }; + + const service = require('./sync'); + const started = await service.maybeRunGitHubSkillSyncForRequest({ + config: { skillSync, config: {} }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + }); + + const requestRunner = mockCreatedRunners[0].runner; + const requestConfig = await mockCreatedRunners[0].deps.getConfig(); + expect(started).toBe(false); + expect(mockCreatedRunners[0].deps.allowServerCredentials).toBe(false); + expect(requestRunner.runOnce).not.toHaveBeenCalled(); + expect(requestConfig.github.runOnStartup).toBe(false); + expect(requestConfig.github.sources[0]).toEqual( + expect.objectContaining({ + id: 'tenant-skills', + tenantId: 'tenant-a', + }), + ); + }); + + it('does not auto-start request-scoped sync when server credentials are unavailable', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [ + { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }; + mockRunnerStatus = { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + provider: 'github', + sourceId: 'tenant-skills', + status: 'idle', + credentialPresent: false, + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + }, + ], + credentials: [], + }; + + const service = require('./sync'); + const started = await service.maybeRunGitHubSkillSyncForRequest({ + config: { skillSync, config: {} }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + }); + + expect(started).toBe(false); + expect(mockCreatedRunners[0].deps.allowServerCredentials).toBe(false); + expect(mockCreatedRunners[0].runner.runOnce).not.toHaveBeenCalled(); + }); + + it('does not start a request-scoped sync for base YAML skillSync config', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [ + { + id: 'base-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }; + mockGetAppConfig.mockResolvedValue({ skillSync }); + + const service = require('./sync'); + const started = await service.maybeRunGitHubSkillSyncForRequest({ + config: { skillSync }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + }); + + expect(started).toBe(false); + expect(mockCreatedRunners).toHaveLength(0); + expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true }); + }); + + it('creates an admin request runner from resolved skillSync config overrides', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [ + { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + tenantId: 'other-tenant', + }, + ], + }, + }; + + const service = require('./sync'); + const runner = service.getGitHubSkillSyncRunnerForRequest({ + config: { skillSync, config: {} }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + skillSyncAllowServerCredentials: true, + }); + const config = await mockCreatedRunners[0].deps.getConfig(); + + expect(runner.runOnce).toBe(mockCreatedRunners[0].runner.runOnce); + expect(runner.getStatus).toBe(mockCreatedRunners[0].runner.getStatus); + expect(mockCreatedRunners[0].deps.allowServerCredentials).toBe(true); + expect(config.github.runOnStartup).toBe(true); + expect(config.github.sources[0]).toEqual( + expect.objectContaining({ id: 'tenant-skills', tenantId: 'tenant-a' }), + ); + }); + + it('preserves base admin runner tenant scope when request config has no nested base copy', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [ + { + id: 'base-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + tenantId: 'base-tenant', + }, + ], + }, + }; + + const service = require('./sync'); + service.initializeGitHubSkillSync({ skillSync }); + service.getGitHubSkillSyncRunnerForRequest({ + config: { skillSync }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + skillSyncAllowServerCredentials: true, + }); + const config = await mockCreatedRunners[1].deps.getConfig(); + + expect(config.github.sources[0]).toEqual( + expect.objectContaining({ id: 'base-skills', tenantId: 'base-tenant' }), + ); + }); + + it('does not allow request-built admin override runners to use server credentials by default', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [ + { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }; + + const service = require('./sync'); + service.getGitHubSkillSyncRunnerForRequest({ + config: { skillSync, config: {} }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + }); + + expect(mockCreatedRunners[0].deps.allowServerCredentials).toBe(false); + }); + + it('does not start a request-scoped sync when the configured source is already running', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }; + mockRunnerStatus = { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + provider: 'github', + sourceId: 'tenant-skills', + status: 'running', + credentialPresent: true, + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + startedAt: new Date(), + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + }, + ], + credentials: [], + }; + + const service = require('./sync'); + const started = await service.maybeRunGitHubSkillSyncForRequest({ + config: { skillSync, config: {} }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + }); + + expect(started).toBe(false); + expect(mockCreatedRunners[0].runner.runOnce).not.toHaveBeenCalled(); + }); + + it('retries a request-scoped sync when a running source status is stale', async () => { + const skillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }; + mockRunnerStatus = { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + provider: 'github', + sourceId: 'tenant-skills', + status: 'running', + credentialPresent: true, + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + startedAt: new Date(Date.now() - 40 * 60 * 1000), + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + }, + ], + credentials: [], + }; + + const service = require('./sync'); + const started = await service.maybeRunGitHubSkillSyncForRequest({ + config: { skillSync, config: {} }, + user: { id: 'user-1', tenantId: 'tenant-a' }, + }); + + expect(started).toBe(true); + expect(mockCreatedRunners[0].runner.runOnce).toHaveBeenCalledTimes(1); + }); + + it('uses the file owner when deleting synced files from storage', async () => { + const deleteFile = jest.fn(async () => undefined); + const ownerId = '507f1f77bcf86cd799439011'; + mockGetAppConfig.mockResolvedValue({ skillSync: undefined, paths: {} }); + mockGetStrategyFunctions.mockReturnValue({ deleteFile }); + + const service = require('./sync'); + service.initializeGitHubSkillSync({ skillSync: undefined }); + await mockRunnerDeps.deleteFile({ + filepath: `/uploads/${ownerId}/file.txt`, + source: 'local', + user: ownerId, + tenantId: 'tenant-a', + }); + + expect(deleteFile).toHaveBeenCalledWith( + expect.objectContaining({ + user: expect.objectContaining({ + id: ownerId, + _id: ownerId, + tenantId: 'tenant-a', + }), + }), + expect.objectContaining({ + user: ownerId, + tenantId: 'tenant-a', + }), + ); + }); + + it('does not force manual sync runs into the system tenant context', async () => { + const { runAsSystem } = require('@librechat/data-schemas'); + mockGetAppConfig.mockResolvedValue({ skillSync: undefined, paths: {} }); + + const service = require('./sync'); + const { runner } = service.initializeGitHubSkillSync({ skillSync: undefined }); + await runner.runOnce(); + + expect(runAsSystem).not.toHaveBeenCalled(); + }); + + it('resolves the access role outside tenant isolation but writes the ACL in context', async () => { + const { runAsSystem } = require('@librechat/data-schemas'); + mockGetAppConfig.mockResolvedValue({ skillSync: undefined, paths: {} }); + mockFindRoleByIdentifier.mockResolvedValue({ + _id: 'role-object-id', + resourceType: 'skill', + permBits: 1, + }); + mockGrantPermission.mockResolvedValue({ _id: 'acl-entry-id' }); + + const service = require('./sync'); + service.initializeGitHubSkillSync({ skillSync: undefined }); + await mockRunnerDeps.grantPermission({ + principalType: 'public', + principalId: null, + resourceType: 'skill', + resourceId: 'skill-id', + accessRoleId: 'skill_viewer', + grantedBy: 'system', + }); + + expect(runAsSystem).toHaveBeenCalledTimes(1); + expect(mockFindRoleByIdentifier).toHaveBeenCalledWith('skill_viewer'); + expect(mockGrantPermission).toHaveBeenCalledWith( + 'public', + null, + 'skill', + 'skill-id', + 1, + 'system', + undefined, + 'role-object-id', + ); + }); + + it('fails the grant when the access role does not exist', async () => { + mockGetAppConfig.mockResolvedValue({ skillSync: undefined, paths: {} }); + mockFindRoleByIdentifier.mockResolvedValue(null); + + const service = require('./sync'); + service.initializeGitHubSkillSync({ skillSync: undefined }); + + await expect( + mockRunnerDeps.grantPermission({ + principalType: 'public', + principalId: null, + resourceType: 'skill', + resourceId: 'skill-id', + accessRoleId: 'skill_viewer', + grantedBy: 'system', + }), + ).rejects.toThrow('Role skill_viewer not found'); + expect(mockGrantPermission).not.toHaveBeenCalled(); + }); +}); diff --git a/librechat.example.yaml b/librechat.example.yaml index 4ab2240ea1..b6fba26da7 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -61,6 +61,37 @@ cache: true # # Recommended for download paths: attach a CloudFront response headers policy # # with X-Content-Type-Options: nosniff and CSP default-src 'none'. +# Skill sync configuration (optional) +# GitHub tokens are referenced from environment variables. Put the token in +# `.env`, then reference it here with `token: '${GITHUB_SKILLS_TOKEN}'`. Use a +# GitHub fine-grained personal access token scoped to the selected repository +# with read-only Contents and Metadata permissions. +# skillSync: +# github: +# enabled: false +# intervalMinutes: 60 +# runOnStartup: true +# sources: +# - id: librechat-skills +# owner: your-org +# repo: your-skills-repo +# ref: main +# paths: +# - skills +# # Number of directory levels below each configured path to scan for +# # `SKILL.md`. Use 2 for repos shaped like `skills//`. +# skillDiscoveryDepth: 2 +# token: '${GITHUB_SKILLS_TOKEN}' +# # Optional. Owns the mirrored skills under the given tenant so they are +# # created and shared within that tenant. Required for visibility when +# # tenant isolation is enabled. Omit for single-tenant deployments. +# # Treat as immutable per source id: changing (or adding/removing) the +# # tenantId later leaves previously mirrored skills in the old tenant, +# # where this source's sync can no longer see or clean them up. To move a +# # source between tenants, delete its mirrored skills in the old tenant +# # first, or use a new source id for the new tenant. +# # tenantId: your-tenant-id + # Custom interface configuration interface: customWelcome: 'Welcome to LibreChat! Enjoy your experience.' @@ -153,23 +184,23 @@ interface: # share: true # public: true # Allows users to toggle "share with everyone" for their links. Whether anonymous access is permitted is controlled by ALLOW_SHARED_LINKS_PUBLIC. # mcpServers: - # Controls user permissions for MCP (Model Context Protocol) server management - # - use: Allow users to use configured MCP servers - # - create: Allow users to create and manage new MCP servers - # - share: Allow users to share MCP servers with other users - # - public: Allow users to share MCP servers publicly (with everyone) + # Controls user permissions for MCP (Model Context Protocol) server management + # - use: Allow users to use configured MCP servers + # - create: Allow users to create and manage new MCP servers + # - share: Allow users to share MCP servers with other users + # - public: Allow users to share MCP servers publicly (with everyone) - # Creation / edit MCP server config Dialog config example - # trustCheckbox: - # label: - # en: 'I understand and I want to continue' - # de: 'Ich verstehe und möchte fortfahren' - # de-DE: 'Ich verstehe und möchte fortfahren' # You can narrow translation to regions like (de-DE or de-CH) - # subLabel: - # en: | - # Librechat hasn't reviewed this MCP server. Attackers may attempt to steal your data or trick the model into taking unintended actions, including destroying data. Learn more. - # de: | - # LibreChat hat diesen MCP-Server nicht überprüft. Angreifer könnten versuchen, Ihre Daten zu stehlen oder das Modell zu unbeabsichtigten Aktionen zu verleiten, einschließlich der Zerstörung von Daten. Mehr erfahren. + # Creation / edit MCP server config Dialog config example + # trustCheckbox: + # label: + # en: 'I understand and I want to continue' + # de: 'Ich verstehe und möchte fortfahren' + # de-DE: 'Ich verstehe und möchte fortfahren' # You can narrow translation to regions like (de-DE or de-CH) + # subLabel: + # en: | + # Librechat hasn't reviewed this MCP server. Attackers may attempt to steal your data or trick the model into taking unintended actions, including destroying data. Learn more. + # de: | + # LibreChat hat diesen MCP-Server nicht überprüft. Angreifer könnten versuchen, Ihre Daten zu stehlen oder das Modell zu unbeabsichtigten Aktionen zu verleiten, einschließlich der Zerstörung von Daten. Mehr erfahren. # Temporary chat retention period in hours (default: 720, min: 1, max: 8760) # temporaryChatRetention: 1 diff --git a/package-lock.json b/package-lock.json index 5211db9ec1..c16d5d06ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46037,7 +46037,7 @@ }, "packages/data-provider": { "name": "librechat-data-provider", - "version": "0.8.503", + "version": "0.8.504", "license": "ISC", "dependencies": { "axios": "^1.16.0", diff --git a/packages/api/src/admin/config.handler.spec.ts b/packages/api/src/admin/config.handler.spec.ts index f3d6da119f..b3f8e9ee3b 100644 --- a/packages/api/src/admin/config.handler.spec.ts +++ b/packages/api/src/admin/config.handler.spec.ts @@ -194,6 +194,29 @@ describe('createAdminConfigHandlers', () => { expect(savedOverrides.interface).toEqual({ modelSelect: false }); }); + it('preserves skillSync sections in admin overrides', async () => { + const { handlers, deps } = createHandlers({ + upsertConfig: jest.fn().mockResolvedValue({ _id: 'c1', configVersion: 1 }), + }); + const req = mockReq({ + params: { principalType: 'role', principalId: 'admin' }, + body: { + overrides: { + skillSync: { github: { enabled: true } }, + interface: { modelSelect: false }, + }, + }, + }); + const res = mockRes(); + + await handlers.upsertConfigOverrides(req, res); + + expect(res.statusCode).toBe(201); + const savedOverrides = deps.upsertConfig.mock.calls[0][3]; + expect(savedOverrides.skillSync).toEqual({ github: { enabled: true } }); + expect(savedOverrides.interface).toEqual({ modelSelect: false }); + }); + it('preserves UI sub-keys in composite permission fields like mcpServers', async () => { const { handlers, deps } = createHandlers({ upsertConfig: jest.fn().mockResolvedValue({ _id: 'c1', configVersion: 1 }), @@ -338,6 +361,24 @@ describe('createAdminConfigHandlers', () => { expect(deps.unsetConfigField).not.toHaveBeenCalled(); }); + it('allows deleting skillSync field paths', async () => { + const { handlers, deps } = createHandlers(); + const req = mockReq({ + params: { principalType: 'role', principalId: 'admin' }, + query: { fieldPath: 'skillSync.github.enabled' }, + }); + const res = mockRes(); + + await handlers.deleteConfigField(req, res); + + expect(res.statusCode).toBe(200); + expect(deps.unsetConfigField).toHaveBeenCalledWith( + 'role', + 'admin', + 'skillSync.github.enabled', + ); + }); + it('allows deleting interface UI field paths', async () => { const { handlers, deps } = createHandlers(); const req = mockReq({ @@ -489,6 +530,27 @@ describe('createAdminConfigHandlers', () => { expect(patchedFields['interface.prompts']).toBeUndefined(); }); + it('preserves skillSync field entries in patches', async () => { + const { handlers, deps } = createHandlers(); + const req = mockReq({ + params: { principalType: 'role', principalId: 'admin' }, + body: { + entries: [ + { fieldPath: 'skillSync.github.enabled', value: true }, + { fieldPath: 'interface.modelSelect', value: false }, + ], + }, + }); + const res = mockRes(); + + await handlers.patchConfigField(req, res); + + expect(res.statusCode).toBe(200); + const patchedFields = deps.patchConfigFields.mock.calls[0][3]; + expect(patchedFields['skillSync.github.enabled']).toBe(true); + expect(patchedFields['interface.modelSelect']).toBe(false); + }); + it('blocks peoplePicker permission sub-key paths', async () => { const { handlers, deps } = createHandlers(); const req = mockReq({ diff --git a/packages/api/src/admin/config.ts b/packages/api/src/admin/config.ts index d8e1094db0..1a79e01384 100644 --- a/packages/api/src/admin/config.ts +++ b/packages/api/src/admin/config.ts @@ -1,5 +1,6 @@ import { logger } from '@librechat/data-schemas'; import { + BASE_ONLY_CONFIG_SECTIONS, PrincipalType, PrincipalModel, INTERFACE_PERMISSION_FIELDS, @@ -15,6 +16,7 @@ import type { ServerRequest } from '~/types/http'; const UNSAFE_SEGMENTS = /(?:^|\.)(__[\w]*|constructor|prototype)(?:\.|$)/; const MAX_PATCH_ENTRIES = 100; const DEFAULT_PRIORITY = 10; +const BASE_ONLY_OVERRIDE_SECTIONS = new Set(BASE_ONLY_CONFIG_SECTIONS); export function isValidFieldPath(path: string): boolean { return ( @@ -31,6 +33,10 @@ export function getTopLevelSection(fieldPath: string): string { return fieldPath.split('.')[0]; } +function isBaseOnlyFieldPath(fieldPath: string): boolean { + return BASE_ONLY_OVERRIDE_SECTIONS.has(getTopLevelSection(fieldPath)); +} + /** * Returns true if `fieldPath` targets an interface permission field or permission sub-key. * @@ -316,7 +322,17 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): { return res.status(403).json({ error: 'Insufficient permissions' }); } - let filteredOverrides = overrides; + const filteredOverrides = { + ...(overrides as Record), + } as Partial; + for (const section of BASE_ONLY_OVERRIDE_SECTIONS) { + if (section in filteredOverrides) { + delete (filteredOverrides as Record)[section]; + logger.warn( + `[adminConfig] Stripping base-only config section "${section}" - configure it in librechat.yaml instead`, + ); + } + } const iface = (overrides as Record).interface; if (iface != null && typeof iface === 'object' && !Array.isArray(iface)) { const filteredIface: Record = {}; @@ -345,7 +361,6 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): { ); } } - filteredOverrides = { ...(overrides as Record) } as Partial; if (Object.keys(filteredIface).length > 0) { (filteredOverrides as Record).interface = filteredIface; } else { @@ -436,6 +451,12 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): { } const validEntries = entries.filter((entry) => { + if (isBaseOnlyFieldPath(entry.fieldPath)) { + logger.warn( + `[adminConfig] Stripping base-only config field "${entry.fieldPath}" - configure it in librechat.yaml instead`, + ); + return false; + } if (isInterfacePermissionPath(entry.fieldPath)) { logger.warn( `[adminConfig] Stripping interface permission field "${entry.fieldPath}" — use role permissions instead`, @@ -608,6 +629,13 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): { }); } + if (isBaseOnlyFieldPath(fieldPath)) { + logger.warn( + `[adminConfig] Ignoring delete for base-only config field "${fieldPath}" - configure it in librechat.yaml instead`, + ); + return res.status(200).json({ message: 'No actionable field path provided' }); + } + if (isInterfacePermissionPath(fieldPath)) { logger.warn( `[adminConfig] Ignoring delete for interface permission field "${fieldPath}" — use role permissions instead`, diff --git a/packages/api/src/admin/index.ts b/packages/api/src/admin/index.ts index 038ca23915..90f826d7bf 100644 --- a/packages/api/src/admin/index.ts +++ b/packages/api/src/admin/index.ts @@ -2,9 +2,11 @@ export { createAdminConfigHandlers } from './config'; export { createAdminGrantsHandlers } from './grants'; export { createAdminGroupsHandlers } from './groups'; export { createAdminRolesHandlers } from './roles'; +export { createAdminSkillsSyncAccess, createAdminSkillsSyncHandlers } from './skills'; export { createAdminUsersHandlers } from './users'; export type { AdminConfigDeps } from './config'; export type { AdminGrantsDeps, GrantPrincipalType } from './grants'; export type { AdminGroupsDeps } from './groups'; export type { AdminRolesDeps } from './roles'; +export type { AdminSkillSyncAccessDeps, AdminSkillSyncDeps } from './skills'; export type { AdminUsersDeps } from './users'; diff --git a/packages/api/src/admin/skills.spec.ts b/packages/api/src/admin/skills.spec.ts new file mode 100644 index 0000000000..43312c872e --- /dev/null +++ b/packages/api/src/admin/skills.spec.ts @@ -0,0 +1,363 @@ +import { SystemCapabilities } from '@librechat/data-schemas'; +import type { NextFunction, Response } from 'express'; +import { createAdminSkillsSyncAccess, createAdminSkillsSyncHandlers } from './skills'; + +function createResponse() { + const res = { + status: jest.fn().mockReturnThis(), + json: jest.fn().mockReturnThis(), + }; + return res as unknown as Response & { + status: jest.Mock; + json: jest.Mock; + }; +} + +function createNext(): NextFunction & jest.Mock { + return jest.fn() as NextFunction & jest.Mock; +} + +function createHandlers({ + statusErrorCode, + statusErrorMessage, +}: { statusErrorCode?: string; statusErrorMessage?: string } = {}) { + const runner = { + getStatus: jest.fn(async () => ({ + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + provider: 'github' as const, + sourceId: 'tenant-skills', + tenantId: 'tenant-a', + status: 'idle' as const, + credentialKey: 'github-skills-prod', + credentialPresent: true, + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + errorCode: statusErrorCode, + errorMessage: statusErrorMessage, + startedAt: undefined, + finishedAt: undefined, + lastSuccessAt: undefined, + lastFailureAt: undefined, + createdAt: undefined, + updatedAt: undefined, + }, + ], + credentials: [ + { + provider: 'github' as const, + credentialKey: 'github-skills-prod', + credentialPresent: true, + tokenFingerprint: 'abc123', + }, + ], + fineGrainedTokenRecommendation: 'Use a fine-grained token.', + })), + runOnce: jest.fn(async () => ({ + status: 'completed' as const, + sources: [ + { + provider: 'github' as const, + sourceId: 'tenant-skills', + tenantId: 'tenant-a', + status: 'succeeded' as const, + credentialKey: 'github-skills-prod', + credentialPresent: true, + syncedSkillCount: 1, + syncedFileCount: 2, + deletedSkillCount: 0, + deletedFileCount: 0, + errorCode: statusErrorCode, + errorMessage: statusErrorMessage, + startedAt: undefined, + finishedAt: undefined, + lastSuccessAt: undefined, + lastFailureAt: undefined, + createdAt: undefined, + updatedAt: undefined, + }, + ], + })), + }; + const handlers = createAdminSkillsSyncHandlers({ + runner, + upsertCredential: jest.fn(), + deleteCredential: jest.fn(), + }); + return { handlers, runner }; +} + +describe('createAdminSkillsSyncHandlers', () => { + it('omits credential summaries and source credential metadata for tenant-scoped status reads', async () => { + const { handlers } = createHandlers(); + const res = createResponse(); + + await handlers.getSyncStatus({ skillSyncCanReadCredentials: false } as never, res); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ + credentials: [], + sources: [ + expect.objectContaining({ + credentialKey: undefined, + credentialPresent: false, + owner: undefined, + repo: undefined, + ref: undefined, + paths: undefined, + }), + ], + }), + ); + }); + + it('redacts credential-related errors from tenant-scoped status reads', async () => { + const { handlers } = createHandlers({ + statusErrorCode: 'MISSING_CREDENTIAL', + statusErrorMessage: 'Missing GitHub token environment variable "GITHUB_SKILLS_TOKEN"', + }); + const res = createResponse(); + + await handlers.getSyncStatus({ skillSyncCanReadCredentials: false } as never, res); + + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ + sources: [ + expect.objectContaining({ + errorCode: 'MISSING_CREDENTIAL', + errorMessage: 'GitHub skill sync credentials are not available', + }), + ], + }), + ); + }); + + it('includes credential summaries and source credential metadata for platform status reads', async () => { + const { handlers } = createHandlers({ + statusErrorCode: 'MISSING_CREDENTIAL', + statusErrorMessage: 'Missing GitHub credential "github-skills-prod"', + }); + const res = createResponse(); + + await handlers.getSyncStatus({ skillSyncCanReadCredentials: true } as never, res); + + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ + credentials: [expect.objectContaining({ credentialKey: 'github-skills-prod' })], + sources: [ + expect.objectContaining({ + credentialKey: 'github-skills-prod', + credentialPresent: true, + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + errorMessage: 'Missing GitHub credential "github-skills-prod"', + }), + ], + }), + ); + }); + + it('omits source credential metadata from tenant-scoped manual run responses', async () => { + const { handlers } = createHandlers({ + statusErrorCode: 'MISSING_CREDENTIAL', + statusErrorMessage: 'Missing GitHub credential "github-skills-prod"', + }); + const res = createResponse(); + + await handlers.runSync( + { + skillSyncAllowServerCredentials: true, + skillSyncCanReadCredentials: false, + } as never, + res, + ); + + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ + sources: [ + expect.objectContaining({ + credentialKey: undefined, + credentialPresent: false, + errorMessage: 'GitHub skill sync credentials are not available', + }), + ], + }), + ); + }); +}); + +describe('createAdminSkillsSyncAccess', () => { + const baseSkillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'base-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }; + + function createAccess({ + hasCapability = jest.fn().mockResolvedValue(true), + getAppConfig = jest.fn().mockResolvedValue({ skillSync: undefined }), + }: { + hasCapability?: jest.Mock; + getAppConfig?: jest.Mock; + } = {}) { + return { + access: createAdminSkillsSyncAccess({ getAppConfig, hasCapability }), + getAppConfig, + hasCapability, + }; + } + + it('attaches the base skill sync config for override comparison', async () => { + const getAppConfig = jest.fn().mockResolvedValue({ skillSync: baseSkillSync }); + const { access } = createAccess({ getAppConfig }); + const req = { config: { skillSync: undefined, config: { endpoints: {} } } }; + const res = createResponse(); + const next = createNext(); + + await access.attachBaseSkillSyncConfig(req as never, res, next); + + expect(getAppConfig).toHaveBeenCalledWith({ baseOnly: true }); + expect(req.config.config).toEqual({ endpoints: {}, skillSync: baseSkillSync }); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('marks credential metadata hidden for tenant-scoped status reads', async () => { + const hasCapability = jest.fn( + async (user: { tenantId?: string }, capability: string): Promise => { + if (capability === SystemCapabilities.READ_SKILLS) { + return Boolean(user.tenantId); + } + return true; + }, + ); + const { access } = createAccess({ hasCapability }); + const req = { user: { id: 'user-1', role: 'ADMIN', tenantId: 'tenant-a' } }; + const res = createResponse(); + const next = createNext(); + + await access.requireReadSkills(req as never, res, next); + await access.attachCredentialReadAccess(req as never, res, next); + + expect(req).toMatchObject({ + skillSyncCanReadCredentials: false, + skillSyncAllowServerCredentials: false, + }); + expect(hasCapability).toHaveBeenCalledWith( + { id: 'user-1', role: 'ADMIN', tenantId: 'tenant-a' }, + SystemCapabilities.READ_SKILLS, + ); + expect(hasCapability).toHaveBeenCalledWith( + { id: 'user-1', role: 'ADMIN' }, + SystemCapabilities.READ_SKILLS, + ); + }); + + it('prevents tenant admins from running overrides that require server credentials', async () => { + const tenantSkillSync = { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }; + const hasCapability = jest.fn( + async (user: { tenantId?: string }, capability: string): Promise => { + if (capability === SystemCapabilities.MANAGE_SKILLS) { + return Boolean(user.tenantId); + } + return true; + }, + ); + const { access } = createAccess({ hasCapability }); + const req = { + user: { id: 'user-1', role: 'ADMIN', tenantId: 'tenant-a' }, + config: { skillSync: tenantSkillSync, config: {} }, + }; + const res = createResponse(); + const next = createNext(); + + await access.requireSyncRunCapability(req as never, res, next); + + expect(res.status).toHaveBeenCalledWith(403); + expect(res.json).toHaveBeenCalledWith({ + message: 'Tenant-scoped manual skill sync requires platform credential access', + }); + expect(next).not.toHaveBeenCalled(); + }); + + it('prevents tenant admins from manually running base skill sync config', async () => { + const hasCapability = jest.fn( + async (user: { tenantId?: string }, capability: string): Promise => { + if (capability === SystemCapabilities.MANAGE_SKILLS) { + return Boolean(user.tenantId); + } + return true; + }, + ); + const { access } = createAccess({ hasCapability }); + const req = { + user: { id: 'user-1', role: 'ADMIN', tenantId: 'tenant-a' }, + config: { skillSync: baseSkillSync, config: { skillSync: baseSkillSync } }, + }; + const res = createResponse(); + const next = createNext(); + + await access.requireSyncRunCapability(req as never, res, next); + + expect(res.status).toHaveBeenCalledWith(403); + expect(res.json).toHaveBeenCalledWith({ message: 'Forbidden' }); + expect(next).not.toHaveBeenCalled(); + }); + + it('allows platform admins to manually run base skill sync config with server credentials', async () => { + const { access } = createAccess(); + const req = { + user: { id: 'user-1', role: 'ADMIN', tenantId: 'tenant-a' }, + config: { skillSync: baseSkillSync, config: { skillSync: baseSkillSync } }, + }; + const res = createResponse(); + const next = createNext(); + + await access.requireSyncRunCapability(req as never, res, next); + + expect(req).toMatchObject({ + skillSyncAllowServerCredentials: true, + skillSyncCanReadCredentials: true, + }); + expect(next).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/admin/skills.ts b/packages/api/src/admin/skills.ts new file mode 100644 index 0000000000..4588d0091e --- /dev/null +++ b/packages/api/src/admin/skills.ts @@ -0,0 +1,390 @@ +import { SystemCapabilities } from '@librechat/data-schemas'; +import { skillSyncConfigSchema } from 'librechat-data-provider'; +import type { + TGitHubSkillSyncStatusResponse, + TGitHubSkillSyncSourceStatus, + TGitHubSkillSyncCredentialSummary, + TGitHubSkillSyncManualRunResponse, + SkillSyncConfig, +} from 'librechat-data-provider'; +import type { + ISkillSyncStatus, + SkillSyncProvider, + SkillSyncCredentialSummary, + UpsertSkillSyncCredentialInput, + SystemCapability, +} from '@librechat/data-schemas'; +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import type { Types } from 'mongoose'; +import type { GitHubSkillSyncRunner } from '~/skills/sync'; + +export type AdminSkillsRequest = Request & { + user?: { + _id?: Types.ObjectId; + id?: string; + }; + skillSyncAllowServerCredentials?: boolean; + skillSyncCanReadCredentials?: boolean; +}; + +type SkillSyncConfigContainer = { + skillSync?: unknown; + config?: { + skillSync?: unknown; + } & Record; +} & Record; + +type AdminSkillSyncAccessRequest = Request & { + user?: { + _id?: Types.ObjectId | { toString(): string }; + id?: string; + role?: string; + tenantId?: string; + }; + config?: SkillSyncConfigContainer; + skillSyncAllowServerCredentials?: boolean; + skillSyncCanReadCredentials?: boolean; +}; + +type SkillSyncCapabilityUser = { + id: string; + role: string; + tenantId?: string; +}; + +export type AdminSkillSyncDeps = { + runner?: GitHubSkillSyncRunner; + getRunner?: (req: Request) => GitHubSkillSyncRunner; + upsertCredential: (input: UpsertSkillSyncCredentialInput) => Promise; + deleteCredential: ( + provider: SkillSyncProvider, + credentialKey: string, + ) => Promise<{ deleted: boolean }>; +}; + +export type AdminSkillSyncAccessDeps = { + getAppConfig: (options: { baseOnly: true }) => Promise<{ skillSync?: unknown } | undefined>; + hasCapability: (user: SkillSyncCapabilityUser, capability: SystemCapability) => Promise; +}; + +type AdminSkillsSyncHandler = (req: AdminSkillsRequest, res: Response) => Promise; + +export type AdminSkillsSyncHandlers = { + getSyncStatus: AdminSkillsSyncHandler; + runSync: AdminSkillsSyncHandler; + setCredential: AdminSkillsSyncHandler; + deleteCredential: (req: Request, res: Response) => Promise; +}; + +export type AdminSkillsSyncAccess = { + attachBaseSkillSyncConfig: RequestHandler; + attachCredentialReadAccess: RequestHandler; + requireReadSkills: RequestHandler; + requirePlatformManageSkills: RequestHandler; + requireSyncRunCapability: RequestHandler; +}; + +const CREDENTIAL_KEY_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/; + +function toIso(date: Date | undefined): string | undefined { + return date ? date.toISOString() : undefined; +} + +function serializeCredential( + credential: SkillSyncCredentialSummary, +): TGitHubSkillSyncCredentialSummary { + return { + provider: credential.provider, + credentialKey: credential.credentialKey, + credentialPresent: credential.credentialPresent, + tokenFingerprint: credential.tokenFingerprint, + createdAt: toIso(credential.createdAt), + updatedAt: toIso(credential.updatedAt), + }; +} + +function isCredentialError(status: ISkillSyncStatus): boolean { + if (status.errorCode === 'MISSING_CREDENTIAL') { + return true; + } + return /credential|token environment variable|server github credentials/i.test( + status.errorMessage ?? '', + ); +} + +function serializeErrorMessage( + status: ISkillSyncStatus, + { includeCredentialMetadata }: { includeCredentialMetadata: boolean }, +): string | undefined { + if (includeCredentialMetadata || !isCredentialError(status)) { + return status.errorMessage; + } + return 'GitHub skill sync credentials are not available'; +} + +function serializeSourceStatus( + status: ISkillSyncStatus & { credentialPresent?: boolean }, + { includeCredentialMetadata = true }: { includeCredentialMetadata?: boolean } = {}, +): TGitHubSkillSyncSourceStatus { + const includePrivateSourceMetadata = includeCredentialMetadata; + return { + provider: status.provider, + sourceId: status.sourceId, + tenantId: status.tenantId, + status: status.status, + credentialKey: includeCredentialMetadata ? status.credentialKey : undefined, + credentialPresent: includeCredentialMetadata ? (status.credentialPresent ?? false) : false, + owner: includePrivateSourceMetadata ? status.owner : undefined, + repo: includePrivateSourceMetadata ? status.repo : undefined, + ref: includePrivateSourceMetadata ? status.ref : undefined, + paths: includePrivateSourceMetadata ? status.paths : undefined, + startedAt: toIso(status.startedAt), + finishedAt: toIso(status.finishedAt), + lastSuccessAt: toIso(status.lastSuccessAt), + lastFailureAt: toIso(status.lastFailureAt), + errorCode: status.errorCode, + errorMessage: serializeErrorMessage(status, { includeCredentialMetadata }), + syncedSkillCount: status.syncedSkillCount, + syncedFileCount: status.syncedFileCount, + deletedSkillCount: status.deletedSkillCount, + deletedFileCount: status.deletedFileCount, + createdAt: toIso(status.createdAt), + updatedAt: toIso(status.updatedAt), + }; +} + +function isCredentialKey(value: unknown): value is string { + return typeof value === 'string' && CREDENTIAL_KEY_PATTERN.test(value); +} + +function getUserObjectId(req: AdminSkillsRequest): Types.ObjectId | undefined { + return req.user?._id; +} + +function getCapabilityUser( + req: AdminSkillSyncAccessRequest, + { platformOnly = false }: { platformOnly?: boolean } = {}, +): SkillSyncCapabilityUser | null { + const id = req.user?.id ?? req.user?._id?.toString?.(); + if (!id) { + return null; + } + return { + id, + role: req.user?.role ?? '', + ...(platformOnly ? {} : { tenantId: req.user?.tenantId }), + }; +} + +function parseSkillSyncConfig(raw: unknown): SkillSyncConfig | undefined { + if (!raw || typeof raw !== 'object') { + return undefined; + } + const parsed = skillSyncConfigSchema.safeParse(raw); + return parsed.success ? parsed.data : undefined; +} + +function isSameSkillSyncConfig(left: SkillSyncConfig, right: SkillSyncConfig): boolean { + return JSON.stringify(left ?? null) === JSON.stringify(right ?? null); +} + +function hasResolvedSkillSyncOverride(req: AdminSkillSyncAccessRequest): boolean { + const resolved = parseSkillSyncConfig(req.config?.skillSync); + const base = parseSkillSyncConfig(req.config?.config?.skillSync); + return Boolean(resolved?.github && !isSameSkillSyncConfig(resolved, base)); +} + +function sendInternalServerError(res: Response): void { + res.status(500).json({ message: 'Internal Server Error' }); +} + +export function createAdminSkillsSyncAccess(deps: AdminSkillSyncAccessDeps): AdminSkillsSyncAccess { + async function hasSkillCapability( + req: AdminSkillSyncAccessRequest, + capability: SystemCapability, + { platformOnly = false }: { platformOnly?: boolean } = {}, + ): Promise { + const user = getCapabilityUser(req, { platformOnly }); + if (!user) { + return false; + } + return deps.hasCapability(user, capability); + } + + function requireSkillCapability( + capability: SystemCapability, + { platformOnly = false }: { platformOnly?: boolean } = {}, + ): RequestHandler { + return async ( + req: AdminSkillSyncAccessRequest, + res: Response, + next: NextFunction, + ): Promise => { + try { + const user = getCapabilityUser(req, { platformOnly }); + if (!user) { + res.status(401).json({ message: 'Authentication required' }); + return; + } + if (await deps.hasCapability(user, capability)) { + next(); + return; + } + res.status(403).json({ message: 'Forbidden' }); + } catch { + sendInternalServerError(res); + } + }; + } + + const attachBaseSkillSyncConfig: RequestHandler = async ( + req: AdminSkillSyncAccessRequest, + res: Response, + next: NextFunction, + ): Promise => { + try { + const baseConfig = await deps.getAppConfig({ baseOnly: true }); + const existingConfig = req.config ?? {}; + req.config = { + ...existingConfig, + config: { + ...(existingConfig.config ?? {}), + skillSync: baseConfig?.skillSync, + }, + }; + next(); + } catch { + sendInternalServerError(res); + } + }; + + const attachCredentialReadAccess: RequestHandler = async ( + req: AdminSkillSyncAccessRequest, + res: Response, + next: NextFunction, + ): Promise => { + try { + const canReadCredentials = await hasSkillCapability(req, SystemCapabilities.READ_SKILLS, { + platformOnly: true, + }); + req.skillSyncCanReadCredentials = canReadCredentials; + req.skillSyncAllowServerCredentials = canReadCredentials; + next(); + } catch { + sendInternalServerError(res); + } + }; + + const requireSyncRunCapability: RequestHandler = async ( + req: AdminSkillSyncAccessRequest, + res: Response, + next: NextFunction, + ): Promise => { + try { + const canManagePlatform = await hasSkillCapability(req, SystemCapabilities.MANAGE_SKILLS, { + platformOnly: true, + }); + if (canManagePlatform) { + req.skillSyncAllowServerCredentials = true; + req.skillSyncCanReadCredentials = true; + next(); + return; + } + if ( + hasResolvedSkillSyncOverride(req) && + (await hasSkillCapability(req, SystemCapabilities.MANAGE_SKILLS)) + ) { + res.status(403).json({ + message: 'Tenant-scoped manual skill sync requires platform credential access', + }); + return; + } + res.status(403).json({ message: 'Forbidden' }); + } catch { + sendInternalServerError(res); + } + }; + + return { + attachBaseSkillSyncConfig, + attachCredentialReadAccess, + requireReadSkills: requireSkillCapability(SystemCapabilities.READ_SKILLS), + requirePlatformManageSkills: requireSkillCapability(SystemCapabilities.MANAGE_SKILLS, { + platformOnly: true, + }), + requireSyncRunCapability, + }; +} + +export function createAdminSkillsSyncHandlers(deps: AdminSkillSyncDeps): AdminSkillsSyncHandlers { + function getRunner(req: Request): GitHubSkillSyncRunner { + const runner = deps.getRunner?.(req) ?? deps.runner; + if (!runner) { + throw new Error('GitHub skill sync runner is not configured'); + } + return runner; + } + + async function getSyncStatus(req: AdminSkillsRequest, res: Response) { + const includeCredentialMetadata = req.skillSyncCanReadCredentials !== false; + const status = await getRunner(req).getStatus(); + const response: TGitHubSkillSyncStatusResponse = { + enabled: status.enabled, + intervalMinutes: status.intervalMinutes, + runOnStartup: status.runOnStartup, + sources: status.sources.map((source) => + serializeSourceStatus(source, { includeCredentialMetadata }), + ), + credentials: includeCredentialMetadata ? status.credentials.map(serializeCredential) : [], + fineGrainedTokenRecommendation: status.fineGrainedTokenRecommendation, + }; + return res.status(200).json(response); + } + + async function runSync(req: AdminSkillsRequest, res: Response) { + const includeCredentialMetadata = req.skillSyncCanReadCredentials === true; + const result = await getRunner(req).runOnce(); + const response: TGitHubSkillSyncManualRunResponse = { + status: result.status, + message: result.message, + sources: result.sources.map((source) => + serializeSourceStatus(source, { includeCredentialMetadata }), + ), + }; + return res.status(result.status === 'skipped' ? 202 : 200).json(response); + } + + async function setCredential(req: AdminSkillsRequest, res: Response) { + const { credentialKey } = req.params; + if (!isCredentialKey(credentialKey)) { + return res.status(400).json({ error: 'Invalid credential key' }); + } + const token = (req.body as { token?: unknown } | undefined)?.token; + if (typeof token !== 'string' || token.trim().length === 0) { + return res.status(400).json({ error: 'GitHub token is required' }); + } + const credential = await deps.upsertCredential({ + provider: 'github', + credentialKey, + token: token.trim(), + userId: getUserObjectId(req), + }); + return res.status(200).json(serializeCredential(credential)); + } + + async function deleteCredential(req: Request, res: Response) { + const { credentialKey } = req.params; + if (!isCredentialKey(credentialKey)) { + return res.status(400).json({ error: 'Invalid credential key' }); + } + const result = await deps.deleteCredential('github', credentialKey); + return res.status(200).json({ credentialKey, deleted: result.deleted }); + } + + return { + getSyncStatus, + runSync, + setCredential, + deleteCredential, + }; +} diff --git a/packages/api/src/skills/__tests__/import.test.ts b/packages/api/src/skills/__tests__/import.test.ts index 391c164d73..55416967d2 100644 --- a/packages/api/src/skills/__tests__/import.test.ts +++ b/packages/api/src/skills/__tests__/import.test.ts @@ -67,6 +67,20 @@ function mockZipRequest(buffer: Buffer): ImportRequest { } as unknown as ImportRequest; } +function mockMarkdownRequest(content: string, originalname = 'bad-frontmatter.md'): ImportRequest { + return { + user: { + id: 'user-1', + _id: new Types.ObjectId(), + username: 'tester', + }, + file: { + originalname, + buffer: Buffer.from(content), + }, + } as unknown as ImportRequest; +} + function importSummary(body: unknown): ImportSummary { return (body as { _importSummary: ImportSummary })._importSummary; } @@ -90,6 +104,12 @@ async function zipWithAdditionalFiles(fileCount: number, fileBytes: number): Pro return zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }); } +async function zipWithSkillMarkdown(skillMarkdown: string): Promise { + const zip = new JSZip(); + zip.file('SKILL.md', skillMarkdown); + return zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }); +} + describe('parseFrontmatter', () => { it('extracts name + description from a minimal frontmatter block', () => { const raw = `---\nname: demo\ndescription: A demo skill.\n---\n\n# Body`; @@ -101,6 +121,16 @@ describe('parseFrontmatter', () => { }); }); + it('coerces non-string scalar name and description to strings', () => { + const raw = `---\nname: 123\ndescription: 2024\n---\n\n# Body`; + expect(parseFrontmatter(raw)).toEqual({ + name: '123', + description: '2024', + alwaysApply: undefined, + invalidBooleans: [], + }); + }); + it('extracts always-apply: true', () => { const raw = `---\nname: legal\ndescription: Legal rules.\nalways-apply: true\n---\n\n# Legal body`; expect(parseFrontmatter(raw)).toEqual({ @@ -166,6 +196,20 @@ describe('parseFrontmatter', () => { expect(result.invalidBooleans).toEqual(['alwaysApply']); }); + it('flags legacy YAML boolean aliases as invalid', () => { + const raw = `---\nname: n\ndescription: d\nalways-apply: on\n---\n\nbody`; + const result = parseFrontmatter(raw); + expect(result.alwaysApply).toBeUndefined(); + expect(result.invalidBooleans).toEqual(['always-apply']); + }); + + it.each(['null', '~'])('flags always-apply: %s as invalid', (value) => { + const raw = `---\nname: n\ndescription: d\nalways-apply: ${value}\n---\n\nbody`; + const result = parseFrontmatter(raw); + expect(result.alwaysApply).toBeUndefined(); + expect(result.invalidBooleans).toEqual(['always-apply']); + }); + it('does not flag always-apply when the key is absent', () => { const raw = `---\nname: n\ndescription: d\n---\n\nbody`; expect(parseFrontmatter(raw).invalidBooleans).toEqual([]); @@ -202,6 +246,26 @@ describe('parseFrontmatter', () => { }); }); + it('extracts frontmatter after a BOM and leading blank lines', () => { + const raw = `\uFEFF\n\n---\nname: prologue\ndescription: Has leading whitespace.\n---\n\nbody`; + expect(parseFrontmatter(raw)).toEqual({ + name: 'prologue', + description: 'Has leading whitespace.', + alwaysApply: undefined, + invalidBooleans: [], + }); + }); + + it('does not treat frontmatter scalar lines that start with --- text as closing fences', () => { + const raw = `---\nname: marker\ndescription: 'first\n---not a closing fence\nlast'\nalways-apply: false\n---\n\nbody`; + expect(parseFrontmatter(raw)).toEqual({ + name: 'marker', + description: 'first ---not a closing fence last', + alwaysApply: false, + invalidBooleans: [], + }); + }); + it('returns empty fields when frontmatter is unterminated', () => { const raw = `---\nname: incomplete\n`; expect(parseFrontmatter(raw)).toEqual({ @@ -211,6 +275,18 @@ describe('parseFrontmatter', () => { }); }); + it('returns empty fields when frontmatter YAML is malformed', () => { + const raw = `---\nname: [\n---\n\nbody`; + expect(parseFrontmatter(raw)).toEqual( + expect.objectContaining({ + name: '', + description: '', + invalidBooleans: [], + parseError: expect.any(String), + }), + ); + }); + it('ignores always-apply appearing outside the frontmatter block', () => { const raw = `---\nname: n\ndescription: d\n---\n\nalways-apply: true (but this is in the body)`; const result = parseFrontmatter(raw); @@ -294,4 +370,49 @@ describe('createImportHandler', () => { expect(summary.filesFailed).toBe(3); expect(summary.errors).toHaveLength(3); }); + + it('rejects malformed YAML frontmatter in markdown imports', async () => { + const deps = mockImportDeps(); + const handler = createImportHandler(deps); + const res = mockResponse(); + + await handler(mockMarkdownRequest('---\nname: [\n---\n\nbody'), res); + + expect(res.status).toHaveBeenCalledWith(400); + expect(res.body).toEqual( + expect.objectContaining({ + error: 'Validation failed', + issues: expect.arrayContaining([ + expect.objectContaining({ + field: 'frontmatter', + code: 'INVALID_YAML', + }), + ]), + }), + ); + expect(deps.createSkill).not.toHaveBeenCalled(); + }); + + it('rejects malformed YAML frontmatter in archive imports', async () => { + const deps = mockImportDeps(); + const handler = createImportHandler(deps); + const res = mockResponse(); + const buffer = await zipWithSkillMarkdown('---\nname: [\n---\n\nbody'); + + await handler(mockZipRequest(buffer), res); + + expect(res.status).toHaveBeenCalledWith(400); + expect(res.body).toEqual( + expect.objectContaining({ + error: 'Validation failed', + issues: expect.arrayContaining([ + expect.objectContaining({ + field: 'frontmatter', + code: 'INVALID_YAML', + }), + ]), + }), + ); + expect(deps.createSkill).not.toHaveBeenCalled(); + }); }); diff --git a/packages/api/src/skills/handlers.ts b/packages/api/src/skills/handlers.ts index 3dcda1e151..e2b135c4b0 100644 --- a/packages/api/src/skills/handlers.ts +++ b/packages/api/src/skills/handlers.ts @@ -206,6 +206,7 @@ function serializeSkillFile(file: ISkillFile & { _id: Types.ObjectId }): TSkillF storageKey: file.storageKey, storageRegion: file.storageRegion, source: file.source as TSkillFile['source'], + sourceMetadata: file.sourceMetadata as TSkillFile['sourceMetadata'], mimeType: file.mimeType, bytes: file.bytes, category: file.category, diff --git a/packages/api/src/skills/import.ts b/packages/api/src/skills/import.ts index 21a8d190ef..c93b021593 100644 --- a/packages/api/src/skills/import.ts +++ b/packages/api/src/skills/import.ts @@ -1,7 +1,7 @@ import path from 'path'; import JSZip from 'jszip'; import crypto from 'crypto'; -import { logger, stripYamlTrailingComment } from '@librechat/data-schemas'; +import { logger } from '@librechat/data-schemas'; import { ResourceType, AccessRoleIds, PrincipalType } from 'librechat-data-provider'; import type { ISkill, @@ -12,50 +12,14 @@ import type { } from '@librechat/data-schemas'; import type { Request, Response } from 'express'; import type { Types } from 'mongoose'; +import type { ImportLimits } from './limits'; import { resolveRequestTenantId } from '~/middleware/tenant'; +import { DEFAULT_SKILL_IMPORT_LIMITS } from './limits'; +import { parseSkillMarkdown } from './parse'; -/** Security limits for zip processing. */ -const MAX_ZIP_BYTES = 50 * 1024 * 1024; // 50 MB compressed -const MAX_DECOMPRESSED_BYTES = 500 * 1024 * 1024; // 500 MB total decompressed -const MAX_ENTRIES = 500; -const MAX_SINGLE_FILE_BYTES = 10 * 1024 * 1024; // 10 MB per file const SKILL_MD = 'SKILL.md'; -export interface ImportLimits { - maxZipBytes: number; - maxDecompressedBytes: number; - maxEntries: number; - maxSingleFileBytes: number; -} - -/** Strip surrounding YAML quotes (single or double) from a scalar value. */ -function unquoteYaml(value: string): string { - if ( - value.length >= 2 && - ((value[0] === '"' && value[value.length - 1] === '"') || - (value[0] === "'" && value[value.length - 1] === "'")) - ) { - return value.slice(1, -1); - } - return value; -} - -/** - * Parse a YAML scalar as a strict boolean. Returns `undefined` when the - * value is neither `true` nor `false`. Callers should pre-strip inline - * comments with `stripYamlTrailingComment` when needed; this helper only - * normalizes case / whitespace so the call site stays one-purpose. - */ -function parseBooleanScalar(value: string): boolean | undefined { - const lowered = value.trim().toLowerCase(); - if (lowered === 'true') { - return true; - } - if (lowered === 'false') { - return false; - } - return undefined; -} +export type { ImportLimits } from './limits'; /** * YAML frontmatter parser — extracts the first-class fields LibreChat @@ -81,71 +45,40 @@ export function parseFrontmatter(raw: string): { alwaysApply?: boolean; /** Keys that carried non-boolean values for fields that must be boolean. */ invalidBooleans: string[]; + parseError?: string; } { - const trimmed = raw.trim(); - if (!trimmed.startsWith('---')) { - return { name: '', description: '', invalidBooleans: [] }; + const parsed = parseSkillMarkdown(raw); + const result: { + name: string; + description: string; + alwaysApply?: boolean; + invalidBooleans: string[]; + parseError?: string; + } = { + name: parsed.name, + description: parsed.description, + invalidBooleans: parsed.invalidBooleans, + }; + if (parsed.parseError) { + result.parseError = parsed.parseError; } - const after = trimmed.slice(3); - const closingIdx = after.indexOf('\n---'); - if (closingIdx === -1) { - return { name: '', description: '', invalidBooleans: [] }; + if ('alwaysApply' in parsed) { + result.alwaysApply = parsed.alwaysApply; } - const block = after.slice(0, closingIdx); - let name = ''; - let description = ''; - let alwaysApply: boolean | undefined; - let hasValidCanonicalAlwaysApply = false; - const invalidCanonicalAlwaysApply: string[] = []; - const invalidAliasAlwaysApply: string[] = []; - for (const line of block.split('\n')) { - const colon = line.indexOf(':'); - if (colon === -1) { - continue; - } - const rawKey = line.slice(0, colon).trim(); - const key = rawKey.toLowerCase(); - const rawValue = line.slice(colon + 1).trim(); - if (key === 'name') { - name = unquoteYaml(rawValue); - } else if (key === 'description') { - description = unquoteYaml(rawValue); - } else if (key === 'always-apply' || key === 'alwaysapply') { - const isCanonical = key === 'always-apply'; - // Operate on the raw post-colon text (no outer `unquoteYaml`): a - // line like `always-apply: "true" # note` must have its comment - // stripped BEFORE unquoting. Running `unquoteYaml` on the whole - // line first would miss the quoted branch (the line doesn't end - // in a quote once the comment is attached), and unquoting twice - // would be fragile if `unquoteYaml` ever gains richer YAML-escape - // handling. - const stripped = stripYamlTrailingComment(rawValue).trim(); - if (stripped === '') { - // Empty value or comment-only (`always-apply: # TBD`) — treat as - // absent so mid-edit placeholder states don't reject the save. - continue; - } - const parsed = parseBooleanScalar(unquoteYaml(stripped)); - if (parsed === undefined) { - if (isCanonical) { - invalidCanonicalAlwaysApply.push(rawKey); - } else { - invalidAliasAlwaysApply.push(rawKey); - } - } else { - if (isCanonical) { - hasValidCanonicalAlwaysApply = true; - alwaysApply = parsed; - } else if (!hasValidCanonicalAlwaysApply && invalidCanonicalAlwaysApply.length === 0) { - alwaysApply = parsed; - } - } - } - } - const invalidBooleans = hasValidCanonicalAlwaysApply - ? invalidCanonicalAlwaysApply - : [...invalidCanonicalAlwaysApply, ...invalidAliasAlwaysApply]; - return { name, description, alwaysApply, invalidBooleans }; + return result; +} + +function sendFrontmatterParseError(res: Response, parseError: string) { + return res.status(400).json({ + error: 'Validation failed', + issues: [ + { + field: 'frontmatter', + code: 'INVALID_YAML', + message: `Invalid YAML frontmatter: ${parseError}`, + }, + ], + }); } /** Validates a relative path is safe (no traversal, no absolute paths). */ @@ -267,10 +200,12 @@ export function createImportHandler(deps: ImportSkillDeps) { function getImportLimits(limits?: Partial): ImportLimits { return { - maxZipBytes: limits?.maxZipBytes ?? MAX_ZIP_BYTES, - maxDecompressedBytes: limits?.maxDecompressedBytes ?? MAX_DECOMPRESSED_BYTES, - maxEntries: limits?.maxEntries ?? MAX_ENTRIES, - maxSingleFileBytes: limits?.maxSingleFileBytes ?? MAX_SINGLE_FILE_BYTES, + maxZipBytes: limits?.maxZipBytes ?? DEFAULT_SKILL_IMPORT_LIMITS.maxZipBytes, + maxDecompressedBytes: + limits?.maxDecompressedBytes ?? DEFAULT_SKILL_IMPORT_LIMITS.maxDecompressedBytes, + maxEntries: limits?.maxEntries ?? DEFAULT_SKILL_IMPORT_LIMITS.maxEntries, + maxSingleFileBytes: + limits?.maxSingleFileBytes ?? DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes, }; } @@ -325,7 +260,10 @@ async function handleMarkdown( ) { const content = file.buffer.toString('utf-8'); - const { name, description, alwaysApply, invalidBooleans } = parseFrontmatter(content); + const { name, description, alwaysApply, invalidBooleans, parseError } = parseFrontmatter(content); + if (parseError) { + return sendFrontmatterParseError(res, parseError); + } if (invalidBooleans.length > 0) { return res.status(400).json({ error: 'Validation failed', @@ -436,7 +374,11 @@ async function handleZip( return res.status(400).json({ error: 'SKILL.md exceeds maximum file size' }); } - const { name, description, alwaysApply, invalidBooleans } = parseFrontmatter(skillMdContent); + const { name, description, alwaysApply, invalidBooleans, parseError } = + parseFrontmatter(skillMdContent); + if (parseError) { + return sendFrontmatterParseError(res, parseError); + } if (invalidBooleans.length > 0) { return res.status(400).json({ error: 'Validation failed', diff --git a/packages/api/src/skills/index.ts b/packages/api/src/skills/index.ts index affee245c4..5669d6586b 100644 --- a/packages/api/src/skills/index.ts +++ b/packages/api/src/skills/index.ts @@ -1,5 +1,8 @@ export * from './binary'; export * from './handlers'; export * from './import'; +export * from './limits'; +export * from './parse'; export * from './skillStates'; export * from './deployment'; +export * from './sync'; diff --git a/packages/api/src/skills/limits.ts b/packages/api/src/skills/limits.ts new file mode 100644 index 0000000000..0d7624485a --- /dev/null +++ b/packages/api/src/skills/limits.ts @@ -0,0 +1,13 @@ +export type ImportLimits = { + maxZipBytes: number; + maxDecompressedBytes: number; + maxEntries: number; + maxSingleFileBytes: number; +}; + +export const DEFAULT_SKILL_IMPORT_LIMITS: ImportLimits = { + maxZipBytes: 50 * 1024 * 1024, + maxDecompressedBytes: 500 * 1024 * 1024, + maxEntries: 500, + maxSingleFileBytes: 10 * 1024 * 1024, +}; diff --git a/packages/api/src/skills/parse.ts b/packages/api/src/skills/parse.ts new file mode 100644 index 0000000000..041e87ed5e --- /dev/null +++ b/packages/api/src/skills/parse.ts @@ -0,0 +1,160 @@ +import yaml from 'js-yaml'; + +export type ParsedSkillMarkdown = { + name: string; + description: string; + alwaysApply?: boolean; + frontmatter?: Record; + invalidBooleans: string[]; + parseError?: string; +}; + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function extractFrontmatterBlock(raw: string): string | null { + const normalized = raw.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'); + const firstContentIndex = normalized.search(/\S/); + if (firstContentIndex === -1) { + return null; + } + const content = normalized.slice(firstContentIndex); + const opening = /^---[ \t]*\n/.exec(content); + if (!opening) { + return null; + } + const body = content.slice(opening[0].length); + const closingFence = /(?:^|\n)---[ \t]*(?:\n|$)/.exec(body); + if (!closingFence) { + return null; + } + return body.slice(0, closingFence.index); +} + +function getCaseInsensitive(frontmatter: Record, key: string): unknown { + const entry = Object.entries(frontmatter).find(([candidate]) => candidate.toLowerCase() === key); + return entry?.[1]; +} + +function hasCaseInsensitive(frontmatter: Record, key: string): boolean { + return Object.keys(frontmatter).some((candidate) => candidate.toLowerCase() === key); +} + +function getRawFrontmatterValue(block: string, key: string): string | undefined { + const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const pattern = new RegExp(`^\\s*${escapedKey}\\s*:\\s*(.*)$`, 'i'); + const line = block.split('\n').find((candidate) => pattern.test(candidate)); + const match = line?.match(pattern); + return match?.[1]; +} + +function stripInlineComment(value: string): string { + let quote: '"' | "'" | null = null; + for (let i = 0; i < value.length; i++) { + const char = value[i]; + if ((char === '"' || char === "'") && (!quote || quote === char)) { + quote = quote ? null : char; + continue; + } + if (char === '#' && !quote) { + return value.slice(0, i).trim(); + } + } + return value.trim(); +} + +function normalizeFrontmatterKeys(frontmatter: Record): Record { + return Object.entries(frontmatter).reduce>((acc, [key, value]) => { + const normalizedKey = key.toLowerCase(); + acc[normalizedKey === 'alwaysapply' ? 'alwaysApply' : normalizedKey] = value; + return acc; + }, {}); +} + +function parseBoolean(value: unknown, rawValue?: string): boolean | undefined { + const raw = rawValue === undefined ? undefined : stripInlineComment(rawValue).toLowerCase(); + if (typeof value === 'boolean') { + return raw === 'true' || raw === 'false' ? value : undefined; + } + if (typeof value !== 'string') { + return undefined; + } + const lowered = value.trim().toLowerCase(); + if (lowered === 'true') { + return true; + } + if (lowered === 'false') { + return false; + } + return undefined; +} + +function hasBooleanPlaceholder(rawValue?: string): boolean { + return rawValue !== undefined && stripInlineComment(rawValue).length === 0; +} + +function toScalarString(value: unknown): string { + if (typeof value === 'string') { + return value; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + return ''; +} + +export function parseSkillMarkdown(raw: string): ParsedSkillMarkdown { + const block = extractFrontmatterBlock(raw); + if (!block) { + return { name: '', description: '', invalidBooleans: [] }; + } + let parsed: unknown; + try { + parsed = yaml.load(block); + } catch (error) { + return { + name: '', + description: '', + invalidBooleans: [], + parseError: error instanceof Error ? error.message : 'Invalid YAML frontmatter', + }; + } + const frontmatter = isPlainObject(parsed) ? normalizeFrontmatterKeys(parsed) : {}; + const nameValue = getCaseInsensitive(frontmatter, 'name'); + const descriptionValue = getCaseInsensitive(frontmatter, 'description'); + const whenToUseValue = getCaseInsensitive(frontmatter, 'when-to-use'); + const hasCanonicalAlwaysApply = hasCaseInsensitive(frontmatter, 'always-apply'); + const hasAliasAlwaysApply = hasCaseInsensitive(frontmatter, 'alwaysapply'); + const canonicalAlwaysApplyValue = getCaseInsensitive(frontmatter, 'always-apply'); + const aliasAlwaysApplyValue = getCaseInsensitive(frontmatter, 'alwaysapply'); + const rawCanonicalAlwaysApplyValue = getRawFrontmatterValue(block, 'always-apply'); + const rawAliasAlwaysApplyValue = getRawFrontmatterValue(block, 'alwaysApply'); + const name = toScalarString(nameValue); + let description = ''; + if (descriptionValue !== undefined) { + description = toScalarString(descriptionValue); + } else if (whenToUseValue !== undefined) { + description = toScalarString(whenToUseValue); + } + let alwaysApply: boolean | undefined; + const invalidBooleans: string[] = []; + if (hasCanonicalAlwaysApply) { + alwaysApply = parseBoolean(canonicalAlwaysApplyValue, rawCanonicalAlwaysApplyValue); + if (alwaysApply === undefined && !hasBooleanPlaceholder(rawCanonicalAlwaysApplyValue)) { + invalidBooleans.push('always-apply'); + } + } else if (hasAliasAlwaysApply) { + alwaysApply = parseBoolean(aliasAlwaysApplyValue, rawAliasAlwaysApplyValue); + if (alwaysApply === undefined && !hasBooleanPlaceholder(rawAliasAlwaysApplyValue)) { + invalidBooleans.push('alwaysApply'); + } + } + return { + name, + description, + alwaysApply, + frontmatter, + invalidBooleans, + }; +} diff --git a/packages/api/src/skills/sync/github.spec.ts b/packages/api/src/skills/sync/github.spec.ts new file mode 100644 index 0000000000..d0c8e946a6 --- /dev/null +++ b/packages/api/src/skills/sync/github.spec.ts @@ -0,0 +1,2271 @@ +import crypto from 'crypto'; +import { Types } from 'mongoose'; +import { getTenantId } from '@librechat/data-schemas'; +import type { + ISkill, + ISkillFile, + CreateSkillInput, + CreateSkillResult, + ISkillSyncStatus, + SkillSyncStatusInput, + UpdateSkillInput, + UpdateSkillResult, +} from '@librechat/data-schemas'; +import type { GitHubSkillSyncDeps } from './github'; +import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits'; +import { createGitHubSkillSyncRunner } from './github'; + +function response(body: unknown, status = 200, headers: Record = {}): Response { + const normalizedHeaders = new Map( + Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]), + ); + return { + ok: status >= 200 && status < 300, + status, + headers: { + get: (key: string) => normalizedHeaders.get(key.toLowerCase()) ?? null, + }, + json: async () => body, + } as unknown as Response; +} + +function blob(content: string) { + return { + sha: 'blob-sha', + encoding: 'base64', + size: Buffer.byteLength(content), + content: Buffer.from(content).toString('base64'), + }; +} + +function githubFetch( + skillMarkdown = '---\nname: research\ndescription: Research things\nalways-apply: true\n---\nBody', +): typeof fetch { + return jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'research/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'skill-md-sha', + size: Buffer.byteLength(skillMarkdown), + url: 'https://api.github.test/blob/skill', + }, + { + path: 'research/scripts/run.sh', + mode: '100644', + type: 'blob', + sha: 'file-sha', + size: 7, + url: 'https://api.github.test/blob/file', + }, + ], + }); + } + if (url.includes('/git/blobs/skill-md-sha')) { + return response(blob(skillMarkdown)); + } + if (url.includes('/git/blobs/file-sha')) { + return response(blob('echo ok')); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; +} + +function makeSkill(input: CreateSkillInput): ISkill & { _id: Types.ObjectId } { + return { + _id: new Types.ObjectId(), + name: input.name, + description: input.description, + body: input.body ?? '', + frontmatter: input.frontmatter ?? {}, + author: input.author, + authorName: input.authorName, + version: 1, + source: input.source ?? 'inline', + sourceMetadata: input.sourceMetadata, + fileCount: 0, + alwaysApply: input.alwaysApply ?? false, + tenantId: input.tenantId, + }; +} + +function makeSkillFile( + skill: ISkill & { _id: Types.ObjectId }, + overrides: Partial = {}, +): ISkillFile & { _id: Types.ObjectId } { + return { + _id: new Types.ObjectId(), + skillId: skill._id, + relativePath: 'scripts/run.sh', + file_id: 'old-file-id', + filename: 'run.sh', + filepath: '/uploads/old-file-id__run.sh', + source: 'local', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + commitSha: 'old-commit-sha', + blobSha: 'old-file-sha', + path: 'skills/research/scripts/run.sh', + }, + mimeType: 'application/x-sh', + bytes: 7, + category: 'script', + isExecutable: false, + author: skill.author, + ...overrides, + }; +} + +function makeSourceAuthorId(sourceId = 'librechat-skills', tenantId?: string): Types.ObjectId { + const seed = tenantId ? `github:${sourceId}:${tenantId}` : `github:${sourceId}`; + return new Types.ObjectId(crypto.createHash('sha256').update(seed).digest('hex').slice(0, 24)); +} + +function createDeps( + overrides: Partial = {}, +): GitHubSkillSyncDeps & { statuses: ISkillSyncStatus[] } { + const statuses: ISkillSyncStatus[] = []; + const deps: GitHubSkillSyncDeps & { statuses: ISkillSyncStatus[] } = { + statuses, + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }), + getCredentialToken: jest.fn(async () => 'github_pat_secret'), + getCredentialSummary: jest.fn(async () => ({ + provider: 'github' as const, + credentialKey: 'github-skills-prod', + credentialPresent: true, + tokenFingerprint: 'abc123', + })), + listCredentials: jest.fn(async () => []), + listStatuses: jest.fn(async () => statuses), + upsertStatus: jest.fn(async (input: SkillSyncStatusInput) => { + const status: ISkillSyncStatus = { + provider: input.provider, + sourceId: input.sourceId, + tenantId: input.tenantId, + status: input.status, + credentialKey: input.credentialKey, + owner: input.owner, + repo: input.repo, + ref: input.ref, + paths: input.paths, + startedAt: input.startedAt, + finishedAt: input.finishedAt, + lastSuccessAt: input.status === 'succeeded' ? input.finishedAt : undefined, + lastFailureAt: input.status === 'failed' ? input.finishedAt : undefined, + errorCode: input.errorCode, + errorMessage: input.errorMessage, + syncedSkillCount: input.syncedSkillCount ?? 0, + syncedFileCount: input.syncedFileCount ?? 0, + deletedSkillCount: input.deletedSkillCount ?? 0, + deletedFileCount: input.deletedFileCount ?? 0, + }; + statuses.push(status); + return status; + }), + tryAcquireLock: jest.fn(async () => true), + refreshLock: jest.fn(async () => true), + releaseLock: jest.fn(async () => undefined), + createSkill: jest.fn(async (input: CreateSkillInput): Promise => { + return { skill: makeSkill(input), warnings: [] }; + }), + updateSkill: jest.fn(), + getSkillById: jest.fn(), + findSkillBySourceIdentity: jest.fn(async () => null), + listSkillsBySource: jest.fn(async () => []), + listSkillFiles: jest.fn(async () => []), + getSkillFileByPath: jest.fn(async () => null), + upsertSkillFile: jest.fn(async () => { + return { + _id: new Types.ObjectId(), + skillId: new Types.ObjectId(), + relativePath: 'scripts/run.sh', + file_id: 'file-id', + filename: 'run.sh', + filepath: '/uploads/file-id__run.sh', + source: 'local', + mimeType: 'application/x-sh', + bytes: 7, + category: 'script', + isExecutable: false, + author: new Types.ObjectId(), + } as ISkillFile & { _id: Types.ObjectId }; + }), + deleteSkillFile: jest.fn(async () => ({ deleted: true })), + deleteSkill: jest.fn(async () => ({ deleted: true })), + saveBuffer: jest.fn(async () => ({ filepath: '/uploads/file-id__run.sh', source: 'local' })), + deleteFile: jest.fn(async () => undefined), + grantPermission: jest.fn(async () => undefined), + fetchFn: githubFetch(), + ...overrides, + }; + return deps; +} + +describe('createGitHubSkillSyncRunner', () => { + it('creates a GitHub skill and syncs bundled files from a configured path', async () => { + const deps = createDeps(); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + const fetchedUrls = (deps.fetchFn as unknown as jest.Mock).mock.calls.map( + ([input]: [RequestInfo | URL]) => input.toString(), + ); + + expect(result.status).toBe('completed'); + expect(fetchedUrls.some((url) => url.includes('/git/trees/tree-sha?recursive=1'))).toBe(false); + expect(fetchedUrls.some((url) => url.includes('/git/trees/skills-tree-sha?recursive=1'))).toBe( + true, + ); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'research', + description: 'Research things', + body: expect.stringContaining('Body'), + alwaysApply: true, + source: 'github', + sourceMetadata: expect.objectContaining({ + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + skillBlobSha: 'skill-md-sha', + }), + }), + ); + expect(deps.upsertSkillFile).toHaveBeenCalledWith( + expect.objectContaining({ + relativePath: 'scripts/run.sh', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills/research', + blobSha: 'file-sha', + commitSha: 'commit-sha', + }), + }), + ); + expect(deps.grantPermission).toHaveBeenCalledWith( + expect.objectContaining({ + principalType: 'public', + accessRoleId: 'skill_viewer', + }), + ); + }); + + it('drops an invalid alwaysApply alias when canonical always-apply is valid', async () => { + const deps = createDeps({ + fetchFn: githubFetch( + '---\nname: research\ndescription: Research things\nalways-apply: true\nalwaysApply: yes\n---\nBody', + ), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + alwaysApply: true, + frontmatter: { 'always-apply': true }, + }), + ); + }); + + it('fails duplicate discovered skill names before publishing partial mirrors', async () => { + const duplicateFetch = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'research/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'skill-a-sha', + size: 50, + url: 'https://api.github.test/blob/skill-a', + }, + { + path: 'analysis/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'skill-b-sha', + size: 50, + url: 'https://api.github.test/blob/skill-b', + }, + ], + }); + } + if (url.includes('/git/blobs/skill-a-sha')) { + return response(blob('---\nname: duplicate\ndescription: First\n---\nBody')); + } + if (url.includes('/git/blobs/skill-b-sha')) { + return response(blob('---\nname: duplicate\ndescription: Second\n---\nBody')); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const deps = createDeps({ fetchFn: duplicateFetch }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.createSkill).not.toHaveBeenCalled(); + expect(deps.updateSkill).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'DUPLICATE_SKILL_NAME', + errorMessage: 'GitHub source "librechat-skills" contains multiple skills named "duplicate"', + }), + ); + }); + + it('fails duplicate root and nested skill names before publishing partial mirrors', async () => { + const duplicateFetch = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'SKILL.md', + mode: '100644', + type: 'blob', + sha: 'root-skill-sha', + size: 50, + url: 'https://api.github.test/blob/root-skill', + }, + { + path: 'child/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'child-skill-sha', + size: 50, + url: 'https://api.github.test/blob/child-skill', + }, + ], + }); + } + if (url.includes('/git/blobs/root-skill-sha')) { + return response(blob('---\nname: duplicate\ndescription: Root\n---\nBody')); + } + if (url.includes('/git/blobs/child-skill-sha')) { + return response(blob('---\nname: duplicate\ndescription: Child\n---\nBody')); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const deps = createDeps({ + fetchFn: duplicateFetch, + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: [''], + credentialKey: 'github-skills-prod', + }, + ], + }, + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.createSkill).not.toHaveBeenCalled(); + expect(deps.updateSkill).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'DUPLICATE_SKILL_NAME', + errorMessage: 'GitHub source "librechat-skills" contains multiple skills named "duplicate"', + }), + ); + }); + + it('discovers nested skill roots within the configured discovery depth', async () => { + const skillMarkdown = '---\nname: tdd\ndescription: Test-driven development\n---\nBody'; + const fetchFn = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'engineering/tdd/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'skill-md-sha', + size: Buffer.byteLength(skillMarkdown), + url: 'https://api.github.test/blob/skill', + }, + { + path: 'engineering/tdd/tests.md', + mode: '100644', + type: 'blob', + sha: 'tests-md-sha', + size: 5, + url: 'https://api.github.test/blob/tests', + }, + ], + }); + } + if (url.includes('/git/blobs/skill-md-sha')) { + return response(blob(skillMarkdown)); + } + if (url.includes('/git/blobs/tests-md-sha')) { + return response(blob('tests')); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const deps = createDeps({ + fetchFn, + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'mattpocock-skills', + owner: 'mattpocock', + repo: 'skills', + ref: 'main', + paths: ['skills'], + skillDiscoveryDepth: 2, + credentialKey: 'github-skills-prod', + }, + ], + }, + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'tdd', + sourceMetadata: expect.objectContaining({ + sourceId: 'mattpocock-skills', + upstreamId: 'mattpocock-skills:skills/engineering/tdd', + }), + }), + ); + expect(deps.upsertSkillFile).toHaveBeenCalledWith( + expect.objectContaining({ + relativePath: 'tests.md', + sourceMetadata: expect.objectContaining({ + path: 'skills/engineering/tdd/tests.md', + }), + }), + ); + }); + + it('uses an env-backed source token without loading a stored credential', async () => { + const previousToken = process.env.GITHUB_SKILLS_TOKEN; + process.env.GITHUB_SKILLS_TOKEN = 'github_pat_from_env'; + const getCredentialToken = jest.fn(async () => 'github_pat_from_db'); + const deps = createDeps({ + getCredentialToken, + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + + try { + const status = await runner.getStatus(); + const result = await runner.runOnce(); + + expect(status.sources[0]?.credentialPresent).toBe(true); + expect(result.status).toBe('completed'); + expect(getCredentialToken).not.toHaveBeenCalled(); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + sourceMetadata: expect.objectContaining({ sourceId: 'librechat-skills' }), + }), + ); + } finally { + if (previousToken == null) { + delete process.env.GITHUB_SKILLS_TOKEN; + } else { + process.env.GITHUB_SKILLS_TOKEN = previousToken; + } + } + }); + + it('does not list or resolve server credentials and skips runs when they are disabled', async () => { + const previousToken = process.env.GITHUB_SKILLS_TOKEN; + process.env.GITHUB_SKILLS_TOKEN = 'github_pat_from_env'; + const getCredentialToken = jest.fn(async () => 'github_pat_from_db'); + const listCredentials = jest.fn(async () => [ + { + provider: 'github' as const, + credentialKey: 'github-skills-prod', + credentialPresent: true, + tokenFingerprint: 'abc123', + }, + ]); + const deps = createDeps({ + allowServerCredentials: false, + getCredentialToken, + listCredentials, + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + { + id: 'stored-credential-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + + try { + const status = await runner.getStatus(); + const result = await runner.runOnce(); + + expect(status.credentials).toEqual([]); + expect(status.sources).toEqual([ + expect.objectContaining({ sourceId: 'librechat-skills', credentialPresent: false }), + expect.objectContaining({ + sourceId: 'stored-credential-skills', + credentialPresent: false, + }), + ]); + expect(result.status).toBe('skipped'); + expect(result.message).toBe( + 'GitHub skill sync credentials are not available for this runner', + ); + expect(result.sources).toEqual([ + expect.objectContaining({ sourceId: 'librechat-skills', credentialPresent: false }), + expect.objectContaining({ + sourceId: 'stored-credential-skills', + credentialPresent: false, + }), + ]); + expect(listCredentials).not.toHaveBeenCalled(); + expect(getCredentialToken).not.toHaveBeenCalled(); + expect(deps.fetchFn).not.toHaveBeenCalled(); + expect(deps.tryAcquireLock).not.toHaveBeenCalled(); + expect(deps.upsertStatus).not.toHaveBeenCalled(); + } finally { + if (previousToken == null) { + delete process.env.GITHUB_SKILLS_TOKEN; + } else { + process.env.GITHUB_SKILLS_TOKEN = previousToken; + } + } + }); + + it('preserves slash-delimited refs when fetching the GitHub commit', async () => { + const baseFetch = githubFetch(); + const fetchFn = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + expect(url).toContain('/commits/heads/release/2026-05'); + expect(url).not.toContain('heads%2Frelease%2F2026-05'); + } + return baseFetch(input); + }) as unknown as typeof fetch; + const deps = createDeps({ + fetchFn, + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'heads/release/2026-05', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }), + }); + const result = await createGitHubSkillSyncRunner(deps).runOnce(); + + expect(result.status).toBe('completed'); + expect(fetchFn).toHaveBeenCalled(); + }); + + it('runs a tenant-scoped source inside its tenant context and stamps the skill tenantId', async () => { + let observedTenantId: string | undefined = 'unset'; + const deps = createDeps({ + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + tenantId: 'tenant-a', + }, + ], + }, + }), + createSkill: jest.fn(async (input: CreateSkillInput): Promise => { + observedTenantId = getTenantId(); + return { skill: makeSkill(input), warnings: [] }; + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(observedTenantId).toBe('tenant-a'); + expect(deps.findSkillBySourceIdentity).toHaveBeenCalledWith({ + source: 'github', + upstreamId: 'librechat-skills:skills/research', + tenantId: 'tenant-a', + }); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ name: 'research', tenantId: 'tenant-a' }), + ); + expect(deps.upsertStatus).toHaveBeenCalledWith( + expect.objectContaining({ sourceId: 'librechat-skills', tenantId: 'tenant-a' }), + ); + const [lockParams] = (deps.tryAcquireLock as jest.Mock).mock.calls[0]; + expect(lockParams).not.toHaveProperty('tenantId'); + }); + + it('matches stored source status by tenant and source id', async () => { + const deps = createDeps({ + listStatuses: jest.fn(async () => [ + { + provider: 'github', + sourceId: 'librechat-skills', + tenantId: 'tenant-a', + status: 'succeeded', + syncedSkillCount: 1, + syncedFileCount: 2, + deletedSkillCount: 0, + deletedFileCount: 0, + } as ISkillSyncStatus, + { + provider: 'github', + sourceId: 'librechat-skills', + tenantId: 'tenant-b', + status: 'failed', + errorCode: 'OTHER_TENANT', + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + } as ISkillSyncStatus, + ]), + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + tenantId: 'tenant-b', + }, + ], + }, + }), + }); + + const status = await createGitHubSkillSyncRunner(deps).getStatus(); + + expect(status.sources[0]).toEqual( + expect.objectContaining({ + sourceId: 'librechat-skills', + tenantId: 'tenant-b', + status: 'failed', + errorCode: 'OTHER_TENANT', + }), + ); + }); + + it('runs in the ambient context when a source has no configured tenantId', async () => { + let observedTenantId: string | undefined = 'unset'; + const deps = createDeps({ + createSkill: jest.fn(async (input: CreateSkillInput): Promise => { + observedTenantId = getTenantId(); + return { skill: makeSkill(input), warnings: [] }; + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(observedTenantId).toBeUndefined(); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ name: 'research', tenantId: undefined }), + ); + }); + + it('scopes mirror cleanup to the current source and deletes only its absent upstream skills', async () => { + const keptId = new Types.ObjectId(); + const staleId = new Types.ObjectId(); + const existingSkill = (upstreamId: string, _id: Types.ObjectId) => { + const skill = makeSkill({ + name: 'research', + description: 'Research things', + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { provider: 'github', sourceId: 'librechat-skills', upstreamId }, + }); + skill._id = _id; + return skill; + }; + const listSkillsBySource = jest.fn(async () => [ + existingSkill('librechat-skills:skills/research', keptId), + existingSkill('librechat-skills:skills/removed', staleId), + ]); + const deps = createDeps({ listSkillsBySource }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(listSkillsBySource).toHaveBeenCalledWith({ + source: 'github', + sourceId: 'librechat-skills', + }); + expect(deps.deleteSkill).toHaveBeenCalledTimes(1); + expect(deps.deleteSkill).toHaveBeenCalledWith(staleId.toString()); + }); + + it('deletes stale name-conflicting mirrors after file sync and before same-commit renames', async () => { + const staleId = new Types.ObjectId(); + const existingId = new Types.ObjectId(); + const author = makeSourceAuthorId(); + const existingSkill = ( + upstreamId: string, + _id: Types.ObjectId, + name: string, + ): ISkill & { _id: Types.ObjectId } => { + const skill = makeSkill({ + name, + description: `${name} skill`, + body: 'Old body', + author, + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { provider: 'github', sourceId: 'librechat-skills', upstreamId }, + }); + skill._id = _id; + return skill; + }; + const staleSkill = existingSkill('librechat-skills:skills/removed', staleId, 'renamed'); + const syncedSkill = existingSkill('librechat-skills:skills/research', existingId, 'research'); + const existingById = new Map([[existingId.toString(), syncedSkill]]); + const deletedIds = new Set(); + const listSkillsBySource = jest.fn(async () => + [staleSkill, syncedSkill].filter((skill) => !deletedIds.has(skill._id.toString())), + ); + const deleteSkill = jest.fn(async (id: string) => { + deletedIds.add(id); + return { deleted: true }; + }); + const updateSkill = jest.fn( + async ({ + id, + update, + }: { + id: string; + expectedVersion: number; + update: UpdateSkillInput; + }): Promise => { + if (!deletedIds.has(staleId.toString()) && update.name === 'renamed') { + throw new Error('duplicate key'); + } + const skill = existingById.get(id); + if (!skill) { + return { status: 'not_found' as const }; + } + const updated = { ...skill, ...update, version: skill.version + 1 }; + existingById.set(id, updated); + return { status: 'updated' as const, skill: updated, warnings: [] }; + }, + ); + const deps = createDeps({ + fetchFn: githubFetch('---\nname: renamed\ndescription: Renamed skill\n---\nBody'), + findSkillBySourceIdentity: jest.fn(async ({ upstreamId }) => + upstreamId === 'librechat-skills:skills/research' ? syncedSkill : null, + ), + getSkillById: jest.fn(async (id) => existingById.get(id.toString()) ?? null), + listSkillsBySource, + deleteSkill, + updateSkill, + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deleteSkill).toHaveBeenCalledWith(staleId.toString()); + expect(updateSkill).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingId.toString(), + update: expect.objectContaining({ name: 'renamed' }), + }), + ); + expect((deps.upsertSkillFile as jest.Mock).mock.invocationCallOrder[0]).toBeLessThan( + deleteSkill.mock.invocationCallOrder[0], + ); + expect(deleteSkill.mock.invocationCallOrder[0]).toBeLessThan( + updateSkill.mock.invocationCallOrder[0], + ); + }); + + it('does not delete stale name-conflicting mirrors before another skill file sync fails', async () => { + const renamedMarkdown = '---\nname: renamed\ndescription: Renamed skill\n---\nBody'; + const brokenMarkdown = '---\nname: broken\ndescription: Broken skill\n---\nBody'; + const fetchFn = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'research/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'research-skill-sha', + size: Buffer.byteLength(renamedMarkdown), + url: 'https://api.github.test/blob/research-skill', + }, + { + path: 'research/scripts/run.sh', + mode: '100644', + type: 'blob', + sha: 'research-file-sha', + size: 7, + url: 'https://api.github.test/blob/research-file', + }, + { + path: 'broken/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'broken-skill-sha', + size: Buffer.byteLength(brokenMarkdown), + url: 'https://api.github.test/blob/broken-skill', + }, + { + path: 'broken/scripts/run.sh', + mode: '100644', + type: 'blob', + sha: 'broken-file-sha', + size: 7, + url: 'https://api.github.test/blob/broken-file', + }, + ], + }); + } + if (url.includes('/git/blobs/research-skill-sha')) { + return response(blob(renamedMarkdown)); + } + if (url.includes('/git/blobs/broken-skill-sha')) { + return response(blob(brokenMarkdown)); + } + if (url.includes('/git/blobs/research-file-sha')) { + return response(blob('echo ok')); + } + if (url.includes('/git/blobs/broken-file-sha')) { + return response(blob('echo ok')); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const staleId = new Types.ObjectId(); + const existingId = new Types.ObjectId(); + const author = makeSourceAuthorId(); + const makeExisting = ( + upstreamId: string, + _id: Types.ObjectId, + name: string, + ): ISkill & { _id: Types.ObjectId } => { + const skill = makeSkill({ + name, + description: `${name} skill`, + body: 'Old body', + author, + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { provider: 'github', sourceId: 'librechat-skills', upstreamId }, + }); + skill._id = _id; + return skill; + }; + const staleSkill = makeExisting('librechat-skills:skills/removed', staleId, 'renamed'); + const syncedSkill = makeExisting('librechat-skills:skills/research', existingId, 'research'); + const createdIds: string[] = []; + const deleteSkill = jest.fn(async (id: string) => ({ deleted: createdIds.includes(id) })); + const deps = createDeps({ + fetchFn, + findSkillBySourceIdentity: jest.fn(async ({ upstreamId }) => + upstreamId === 'librechat-skills:skills/research' ? syncedSkill : null, + ), + getSkillById: jest.fn(async (id) => + id.toString() === existingId.toString() ? syncedSkill : null, + ), + listSkillsBySource: jest.fn(async () => [staleSkill, syncedSkill]), + createSkill: jest.fn(async (input: CreateSkillInput): Promise => { + const skill = makeSkill(input); + createdIds.push(skill._id.toString()); + return { skill, warnings: [] }; + }), + saveBuffer: jest.fn(async () => { + throw new Error('storage unavailable'); + }), + deleteSkill, + updateSkill: jest.fn(), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deleteSkill).not.toHaveBeenCalledWith(staleId.toString()); + expect(deps.updateSkill).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorMessage: 'storage unavailable', + }), + ); + }); + + it('restores a stale name-conflicting mirror when the rename update fails after deletion', async () => { + const staleId = new Types.ObjectId(); + const existingId = new Types.ObjectId(); + const author = makeSourceAuthorId(); + const makeExisting = ( + upstreamId: string, + _id: Types.ObjectId, + name: string, + ): ISkill & { _id: Types.ObjectId } => { + const skill = makeSkill({ + name, + description: `${name} skill`, + body: 'Old body', + author, + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { provider: 'github', sourceId: 'librechat-skills', upstreamId }, + }); + skill._id = _id; + return skill; + }; + const staleSkill = makeExisting('librechat-skills:skills/removed', staleId, 'renamed'); + const syncedSkill = makeExisting('librechat-skills:skills/research', existingId, 'research'); + const deletedIds = new Set(); + let restoredSkill: (ISkill & { _id: Types.ObjectId }) | undefined; + const createSkill = jest.fn(async (input: CreateSkillInput): Promise => { + restoredSkill = makeSkill(input); + return { skill: restoredSkill, warnings: [] }; + }); + const deleteSkill = jest.fn(async (id: string) => { + deletedIds.add(id); + return { deleted: true }; + }); + const deps = createDeps({ + fetchFn: githubFetch('---\nname: renamed\ndescription: Renamed skill\n---\nBody'), + findSkillBySourceIdentity: jest.fn(async ({ upstreamId }) => + upstreamId === 'librechat-skills:skills/research' ? syncedSkill : null, + ), + getSkillById: jest.fn(async (id) => + id.toString() === existingId.toString() ? syncedSkill : null, + ), + listSkillsBySource: jest.fn(async () => + [staleSkill, syncedSkill].filter((skill) => !deletedIds.has(skill._id.toString())), + ), + createSkill, + deleteSkill, + updateSkill: jest.fn(async () => ({ status: 'conflict' as const, current: syncedSkill })), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deleteSkill).toHaveBeenCalledWith(staleId.toString()); + expect(createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'renamed', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills/removed', + }), + }), + ); + expect(deps.grantPermission).toHaveBeenCalledWith( + expect.objectContaining({ resourceId: restoredSkill?._id }), + ); + }); + + it("does not mirror-delete another tenant's skills from an ambient source run", async () => { + const ambientStaleId = new Types.ObjectId(); + const otherTenantId = new Types.ObjectId(); + const makeExisting = ( + upstreamId: string, + _id: Types.ObjectId, + tenantId?: string, + ): ISkill & { _id: Types.ObjectId } => { + const skill = makeSkill({ + name: 'research', + description: 'Research things', + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { provider: 'github', sourceId: 'librechat-skills', upstreamId }, + }); + skill._id = _id; + skill.tenantId = tenantId; + return skill; + }; + // The configured source is ambient (no tenantId), but listSkillsBySource + // (non-strict) returns a skill owned by tenant-b. It must not be deleted. + const listSkillsBySource = jest.fn(async () => [ + makeExisting('librechat-skills:skills/removed', ambientStaleId, undefined), + makeExisting('librechat-skills:skills/removed', otherTenantId, 'tenant-b'), + ]); + const deps = createDeps({ listSkillsBySource }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.deleteSkill).toHaveBeenCalledTimes(1); + expect(deps.deleteSkill).toHaveBeenCalledWith(ambientStaleId.toString()); + expect(deps.deleteSkill).not.toHaveBeenCalledWith(otherTenantId.toString()); + }); + + it('derives distinct synthetic authors for the same source mirrored into different tenants', async () => { + const authorForTenant = async (tenantId: string): Promise => { + let author = ''; + const deps = createDeps({ + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + tenantId, + }, + ], + }, + }), + createSkill: jest.fn(async (input: CreateSkillInput): Promise => { + author = input.author.toString(); + return { skill: makeSkill(input), warnings: [] }; + }), + }); + await createGitHubSkillSyncRunner(deps).runOnce(); + return author; + }; + + const [authorA, authorB] = [ + await authorForTenant('tenant-a'), + await authorForTenant('tenant-b'), + ]; + expect(authorA).not.toBe(''); + expect(authorA).not.toBe(authorB); + }); + + it('uses distinct synthetic authors so same-named skills can sync from different sources', async () => { + const seenNamesByAuthor = new Set(); + const deps = createDeps({ + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'source-a', + owner: 'LibreChat', + repo: 'skills-a', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + { + id: 'source-b', + owner: 'LibreChat', + repo: 'skills-b', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }), + createSkill: jest.fn(async (input: CreateSkillInput): Promise => { + const key = `${input.name}:${input.author.toString()}`; + if (seenNamesByAuthor.has(key)) { + throw new Error('duplicate key'); + } + seenNamesByAuthor.add(key); + return { skill: makeSkill(input), warnings: [] }; + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + const createCalls = (deps.createSkill as jest.Mock).mock.calls.map( + ([input]: [CreateSkillInput]) => input, + ); + + expect(result.status).toBe('completed'); + expect(createCalls).toHaveLength(2); + expect(createCalls.map((input) => input.name)).toEqual(['research', 'research']); + expect(new Set(createCalls.map((input) => input.author.toString())).size).toBe(2); + }); + + it('marks a source failed and skips mirror deletion when the credential is missing', async () => { + const deps = createDeps({ + getCredentialToken: jest.fn(async () => null), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.listSkillsBySource).not.toHaveBeenCalled(); + expect(deps.deleteSkill).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'MISSING_CREDENTIAL', + }), + ); + }); + + it('marks GitHub secondary rate limits as rate limited instead of auth failures', async () => { + const deps = createDeps({ + fetchFn: jest.fn(async () => + response( + { message: 'You have exceeded a secondary rate limit. Please wait before retrying.' }, + 403, + { 'x-ratelimit-remaining': '42' }, + ), + ) as unknown as typeof fetch, + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'GITHUB_RATE_LIMITED', + }), + ); + }); + + it('keeps non-rate-limit GitHub 403 responses classified as auth failures', async () => { + const deps = createDeps({ + fetchFn: jest.fn(async () => + response({ message: 'Resource not accessible by personal access token' }, 403, { + 'x-ratelimit-remaining': '42', + }), + ) as unknown as typeof fetch, + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'GITHUB_AUTH_FAILED', + }), + ); + }); + + it('marks a source failed and skips mirror deletion when SKILL.md frontmatter is malformed', async () => { + const deps = createDeps({ + fetchFn: githubFetch('---\nname: [\n---\nBody'), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.createSkill).not.toHaveBeenCalled(); + expect(deps.listSkillsBySource).not.toHaveBeenCalled(); + expect(deps.deleteSkill).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'SKILL_PARSE_FAILED', + errorMessage: expect.stringContaining('skills/research/SKILL.md'), + }), + ); + }); + + it('uses a ref-independent upstream identity when updating an existing GitHub skill', async () => { + const existing = makeSkill({ + name: 'research', + description: 'Old description', + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + frontmatter: { 'allowed-tools': ['old-tool'] }, + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const deps = createDeps({ + getConfig: () => ({ + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'release', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }), + findSkillBySourceIdentity: jest.fn(async () => existing), + getSkillById: jest.fn(async () => ({ ...existing, version: existing.version + 1 })), + fetchFn: githubFetch('---\nname: research\ndescription: Research things\n---\nBody'), + updateSkill: jest.fn(async ({ update }) => ({ + status: 'updated' as const, + skill: { + ...existing, + ...update, + version: existing.version + 1, + }, + warnings: [], + })), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.findSkillBySourceIdentity).toHaveBeenCalledWith({ + source: 'github', + upstreamId: 'librechat-skills:skills/research', + tenantId: undefined, + }); + expect(deps.createSkill).not.toHaveBeenCalled(); + expect(deps.updateSkill).toHaveBeenCalledWith( + expect.objectContaining({ + update: expect.objectContaining({ + sourceMetadata: expect.objectContaining({ + ref: 'release', + upstreamId: 'librechat-skills:skills/research', + }), + frontmatter: {}, + }), + }), + ); + }); + + it('ignores source identity matches from a different tenant bucket', async () => { + const otherTenantSkill = makeSkill({ + name: 'research', + description: 'Tenant skill', + author: makeSourceAuthorId('librechat-skills', 'tenant-b'), + authorName: 'GitHub Sync', + frontmatter: {}, + source: 'github', + tenantId: 'tenant-b', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const deps = createDeps({ + findSkillBySourceIdentity: jest.fn(async () => otherTenantSkill), + listSkillsBySource: jest.fn(async () => [otherTenantSkill]), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'research', + tenantId: undefined, + }), + ); + expect(deps.updateSkill).not.toHaveBeenCalled(); + expect(deps.deleteSkill).not.toHaveBeenCalledWith(otherTenantSkill._id.toString()); + }); + + it('does not match still-discovered mirrors as moved skills when new skills sync first', async () => { + const newSkillMarkdown = '---\nname: research\ndescription: New research skill\n---\nNew'; + const renamedSkillMarkdown = '---\nname: renamed\ndescription: Renamed skill\n---\nRenamed'; + const fetchFn = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'new/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'new-skill-sha', + size: Buffer.byteLength(newSkillMarkdown), + url: 'https://api.github.test/blob/new-skill', + }, + { + path: 'research/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'renamed-skill-sha', + size: Buffer.byteLength(renamedSkillMarkdown), + url: 'https://api.github.test/blob/renamed-skill', + }, + ], + }); + } + if (url.includes('/git/blobs/new-skill-sha')) { + return response(blob(newSkillMarkdown)); + } + if (url.includes('/git/blobs/renamed-skill-sha')) { + return response(blob(renamedSkillMarkdown)); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const existing = makeSkill({ + name: 'research', + description: 'Old research skill', + body: 'Old body', + author: makeSourceAuthorId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const deps = createDeps({ + fetchFn, + findSkillBySourceIdentity: jest.fn(async ({ upstreamId }) => + upstreamId === 'librechat-skills:skills/research' ? existing : null, + ), + listSkillsBySource: jest.fn(async () => [existing]), + getSkillById: jest.fn(async (id) => + id.toString() === existing._id.toString() ? existing : null, + ), + updateSkill: jest.fn(async ({ update }) => ({ + status: 'updated' as const, + skill: { ...existing, ...update, version: existing.version + 1 }, + warnings: [], + })), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'research', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills/new', + }), + }), + ); + expect(deps.updateSkill).toHaveBeenCalledWith( + expect.objectContaining({ + id: existing._id.toString(), + update: expect.objectContaining({ + name: 'renamed', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills/research', + }), + }), + }), + ); + }); + + it('reuses a same-named source mirror when a skill moves configured paths', async () => { + const existing = makeSkill({ + name: 'research', + description: 'Old description', + author: makeSourceAuthorId(), + authorName: 'GitHub Sync', + frontmatter: {}, + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/old-research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/old-research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const unchangedFile = makeSkillFile(existing, { + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/old-research', + commitSha: 'old-commit-sha', + blobSha: 'file-sha', + path: 'skills/old-research/scripts/run.sh', + }, + }); + const deps = createDeps({ + findSkillBySourceIdentity: jest.fn(async () => null), + listSkillsBySource: jest.fn(async () => [existing]), + getSkillById: jest.fn(async () => existing), + getSkillFileByPath: jest.fn(async () => unchangedFile), + listSkillFiles: jest.fn(async () => [unchangedFile]), + updateSkill: jest.fn(async ({ update }) => { + Object.assign(existing, update, { version: existing.version + 1 }); + return { status: 'updated' as const, skill: existing, warnings: [] }; + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).not.toHaveBeenCalled(); + expect(deps.updateSkill).toHaveBeenCalledWith( + expect.objectContaining({ + id: existing._id.toString(), + update: expect.objectContaining({ + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills/research', + skillPath: 'skills/research', + }), + }), + }), + ); + expect(deps.deleteSkill).not.toHaveBeenCalled(); + }); + + it('refreshes an existing skill version after file sync before updating metadata', async () => { + const existing = makeSkill({ + name: 'research', + description: 'Old description', + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const afterFileSync = { ...existing, version: existing.version + 2 }; + const deps = createDeps({ + findSkillBySourceIdentity: jest.fn(async () => existing), + getSkillById: jest.fn(async () => afterFileSync), + updateSkill: jest.fn(async ({ expectedVersion, update }) => ({ + status: 'updated' as const, + skill: { + ...afterFileSync, + ...update, + version: expectedVersion + 1, + }, + warnings: [], + })), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.upsertSkillFile).toHaveBeenCalled(); + expect(deps.getSkillById).toHaveBeenCalledWith(existing._id); + expect(deps.updateSkill).toHaveBeenCalledWith( + expect.objectContaining({ + id: existing._id.toString(), + expectedVersion: afterFileSync.version, + }), + ); + }); + + it('treats frontmatter-only edits during sync as conflicts', async () => { + const existing = makeSkill({ + name: 'research', + description: 'Old description', + body: 'Old body', + frontmatter: { 'allowed-tools': ['old-tool'] }, + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const edited = { + ...existing, + version: existing.version + 1, + frontmatter: { 'allowed-tools': ['user-tool'] }, + }; + const deps = createDeps({ + findSkillBySourceIdentity: jest.fn(async () => existing), + getSkillById: jest.fn(async () => edited), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.upsertSkillFile).not.toHaveBeenCalled(); + expect(deps.updateSkill).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'SKILL_CONFLICT', + }), + ); + }); + + it('skips existing skill updates when the upstream package is unchanged', async () => { + const skillMarkdown = '---\nname: research\ndescription: Research things\n---\nBody'; + const existing = makeSkill({ + name: 'research', + description: 'Research things', + body: skillMarkdown, + frontmatter: {}, + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + commitSha: 'old-commit-sha', + skillBlobSha: 'skill-md-sha', + syncedAt: '2026-05-30T00:00:00.000Z', + syncStatus: 'synced', + }, + }) as ISkill & { _id: Types.ObjectId }; + const unchangedFile = makeSkillFile(existing, { + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + commitSha: 'old-commit-sha', + blobSha: 'file-sha', + path: 'skills/research/scripts/run.sh', + }, + }); + const deps = createDeps({ + fetchFn: githubFetch(skillMarkdown), + findSkillBySourceIdentity: jest.fn(async () => existing), + getSkillById: jest.fn(async () => existing), + getSkillFileByPath: jest.fn(async () => unchangedFile), + listSkillFiles: jest.fn(async () => [unchangedFile]), + updateSkill: jest.fn(), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.saveBuffer).not.toHaveBeenCalled(); + expect(deps.upsertSkillFile).not.toHaveBeenCalled(); + expect(deps.updateSkill).not.toHaveBeenCalled(); + expect(deps.grantPermission).toHaveBeenCalled(); + }); + + it('does not mutate existing skill files when permission grant fails', async () => { + const existing = makeSkill({ + name: 'research', + description: 'Old description', + body: 'Old body', + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const deps = createDeps({ + findSkillBySourceIdentity: jest.fn(async () => existing), + getSkillById: jest.fn(async () => existing), + grantPermission: jest.fn(async () => { + throw new Error('permission unavailable'); + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.grantPermission).toHaveBeenCalledWith( + expect.objectContaining({ resourceId: existing._id }), + ); + expect(deps.listSkillFiles).not.toHaveBeenCalled(); + expect(deps.saveBuffer).not.toHaveBeenCalled(); + expect(deps.upsertSkillFile).not.toHaveBeenCalled(); + expect(deps.updateSkill).not.toHaveBeenCalled(); + }); + + it('restores existing skill files when the skill update fails after file sync', async () => { + const existing = makeSkill({ + name: 'research', + description: 'Old description', + body: 'Old body', + author: new Types.ObjectId(), + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata: { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId: 'librechat-skills:skills/research', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + skillPath: 'skills/research', + }, + }) as ISkill & { _id: Types.ObjectId }; + const oldFile = makeSkillFile(existing); + const files = new Map([ + [oldFile.relativePath, oldFile], + ]); + const upsertSkillFile = jest.fn( + async ( + row: Parameters[0], + ): Promise => { + const current = files.get(row.relativePath); + const next = { + _id: current?._id ?? new Types.ObjectId(), + skillId: row.skillId as Types.ObjectId, + relativePath: row.relativePath, + file_id: row.file_id, + filename: row.filename, + filepath: row.filepath, + storageKey: row.storageKey, + storageRegion: row.storageRegion, + source: row.source, + sourceMetadata: row.sourceMetadata, + mimeType: row.mimeType, + bytes: row.bytes, + category: 'script' as const, + isExecutable: row.isExecutable ?? false, + author: row.author, + tenantId: row.tenantId, + }; + files.set(row.relativePath, next); + return next; + }, + ); + const deps = createDeps({ + findSkillBySourceIdentity: jest.fn(async () => existing), + getSkillById: jest.fn(async () => ({ ...existing, version: existing.version + 1 })), + getSkillFileByPath: jest.fn( + async (_skillId, relativePath) => files.get(relativePath) ?? null, + ), + listSkillFiles: jest.fn(async () => Array.from(files.values())), + upsertSkillFile, + deleteSkillFile: jest.fn(async (_skillId, relativePath) => ({ + deleted: files.delete(relativePath), + })), + saveBuffer: jest.fn(async () => ({ + filepath: '/uploads/new-file-id__run.sh', + source: 'local', + })), + deleteFile: jest.fn(async () => undefined), + updateSkill: jest.fn(async () => ({ status: 'conflict' as const, current: existing })), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'SKILL_CONFLICT', + }), + ); + expect(files.get('scripts/run.sh')).toEqual( + expect.objectContaining({ filepath: oldFile.filepath }), + ); + expect(deps.deleteFile).toHaveBeenCalledWith( + expect.objectContaining({ filepath: '/uploads/new-file-id__run.sh' }), + ); + expect(deps.deleteFile).not.toHaveBeenCalledWith( + expect.objectContaining({ filepath: oldFile.filepath }), + ); + }); + + it('preserves credential presence when a manual run is skipped by an active lock', async () => { + const deps = createDeps({ + tryAcquireLock: jest.fn(async () => false), + listCredentials: jest.fn(async () => [ + { + provider: 'github' as const, + credentialKey: 'github-skills-prod', + credentialPresent: true, + tokenFingerprint: 'abc123', + }, + ]), + listStatuses: jest.fn(async () => [ + { + provider: 'github', + sourceId: 'librechat-skills', + status: 'running', + credentialKey: 'github-skills-prod', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + } as ISkillSyncStatus, + ]), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('skipped'); + expect(result.sources).toEqual([ + expect.objectContaining({ + sourceId: 'librechat-skills', + status: 'running', + credentialPresent: true, + }), + ]); + }); + + it('uses a fresh lock owner for each sync run', async () => { + const deps = createDeps({ lockOwner: 'worker-a' }); + const runner = createGitHubSkillSyncRunner(deps); + + await runner.runOnce(); + await runner.runOnce(); + + const lockOwners = (deps.tryAcquireLock as jest.Mock).mock.calls.map( + ([params]: [Parameters[0]]) => params.lockOwner, + ); + const releasedOwners = (deps.releaseLock as jest.Mock).mock.calls.map( + ([params]: [Parameters[0]]) => params.lockOwner, + ); + + expect(lockOwners).toHaveLength(2); + expect(lockOwners[0]).not.toBe(lockOwners[1]); + expect(lockOwners.every((owner) => owner.startsWith('worker-a:'))).toBe(true); + expect(releasedOwners).toEqual(lockOwners); + }); + + it('excludes child skill packages from parent synced files', async () => { + const parentSkillMarkdown = '---\nname: parent\ndescription: Parent skill\n---\nParent'; + const childSkillMarkdown = '---\nname: child\ndescription: Child skill\n---\nChild'; + const fetchFn = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'SKILL.md', + mode: '100644', + type: 'blob', + sha: 'parent-skill-sha', + size: Buffer.byteLength(parentSkillMarkdown), + url: 'https://api.github.test/blob/parent-skill', + }, + { + path: 'parent.txt', + mode: '100644', + type: 'blob', + sha: 'parent-file-sha', + size: 6, + url: 'https://api.github.test/blob/parent-file', + }, + { + path: 'child/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'child-skill-sha', + size: Buffer.byteLength(childSkillMarkdown), + url: 'https://api.github.test/blob/child-skill', + }, + { + path: 'child/child.txt', + mode: '100644', + type: 'blob', + sha: 'child-file-sha', + size: 5, + url: 'https://api.github.test/blob/child-file', + }, + ], + }); + } + if (url.includes('/git/blobs/parent-skill-sha')) { + return response(blob(parentSkillMarkdown)); + } + if (url.includes('/git/blobs/parent-file-sha')) { + return response(blob('parent')); + } + if (url.includes('/git/blobs/child-skill-sha')) { + return response(blob(childSkillMarkdown)); + } + if (url.includes('/git/blobs/child-file-sha')) { + return response(blob('child')); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const deps = createDeps({ fetchFn }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + const fileCalls = (deps.upsertSkillFile as jest.Mock).mock.calls.map( + ([row]: [Parameters[0]]) => row, + ); + + expect(result.status).toBe('completed'); + expect(fileCalls).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relativePath: 'parent.txt', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills', + }), + }), + expect.objectContaining({ + relativePath: 'child.txt', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills/child', + }), + }), + ]), + ); + expect(fileCalls).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relativePath: 'child/SKILL.md', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills', + }), + }), + expect.objectContaining({ + relativePath: 'child/child.txt', + sourceMetadata: expect.objectContaining({ + upstreamId: 'librechat-skills:skills', + }), + }), + ]), + ); + }); + + it('rejects oversized GitHub blobs before downloading file content', async () => { + const skillMarkdown = '---\nname: research\ndescription: Research things\n---\nBody'; + const oversizedBytes = DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes + 1; + const fetchFn = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'research/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'skill-md-sha', + size: Buffer.byteLength(skillMarkdown), + url: 'https://api.github.test/blob/skill', + }, + { + path: 'research/data.bin', + mode: '100644', + type: 'blob', + sha: 'oversized-file-sha', + size: oversizedBytes, + url: 'https://api.github.test/blob/oversized', + }, + ], + }); + } + if (url.includes('/git/blobs/skill-md-sha')) { + return response(blob(skillMarkdown)); + } + if (url.includes('/git/blobs/oversized-file-sha')) { + throw new Error('oversized blob should not be downloaded'); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const deps = createDeps({ fetchFn }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + const fetchedUrls = (fetchFn as unknown as jest.Mock).mock.calls.map( + ([input]: [RequestInfo | URL]) => input.toString(), + ); + + expect(result.status).toBe('failed'); + expect(fetchedUrls.some((url) => url.includes('/git/blobs/oversized-file-sha'))).toBe(false); + expect(deps.createSkill).not.toHaveBeenCalled(); + expect(deps.saveBuffer).not.toHaveBeenCalled(); + expect(deps.listSkillsBySource).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'GITHUB_BLOB_TOO_LARGE', + }), + ); + }); + + it('rejects packages that exceed the skill import entry limit before blob downloads', async () => { + const skillMarkdown = '---\nname: research\ndescription: Research things\n---\nBody'; + const extraFiles = Array.from( + { length: DEFAULT_SKILL_IMPORT_LIMITS.maxEntries }, + (_, index) => ({ + path: `research/files/${index}.txt`, + mode: '100644', + type: 'blob', + sha: `file-${index}-sha`, + size: 1, + url: `https://api.github.test/blob/file-${index}`, + }), + ); + const fetchFn = jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commits/')) { + return response({ sha: 'commit-sha', commit: { tree: { sha: 'tree-sha' } } }); + } + if (url.includes('/git/trees/tree-sha')) { + return response({ + sha: 'tree-sha', + truncated: false, + tree: [ + { + path: 'skills', + mode: '040000', + type: 'tree', + sha: 'skills-tree-sha', + url: 'https://api.github.test/tree/skills', + }, + ], + }); + } + if (url.includes('/git/trees/skills-tree-sha')) { + return response({ + sha: 'skills-tree-sha', + truncated: false, + tree: [ + { + path: 'research/SKILL.md', + mode: '100644', + type: 'blob', + sha: 'skill-md-sha', + size: Buffer.byteLength(skillMarkdown), + url: 'https://api.github.test/blob/skill', + }, + ...extraFiles, + ], + }); + } + if (url.includes('/git/blobs/')) { + throw new Error('blob should not be downloaded'); + } + return response({ message: 'not found' }, 404); + }) as unknown as typeof fetch; + const deps = createDeps({ fetchFn }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.createSkill).not.toHaveBeenCalled(); + expect(deps.saveBuffer).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'GITHUB_TOO_MANY_FILES', + }), + ); + }); + + it('rolls back a newly created skill when file sync fails before publishing', async () => { + const deps = createDeps({ + saveBuffer: jest.fn(async () => { + throw new Error('storage unavailable'); + }), + }); + const runner = createGitHubSkillSyncRunner(deps); + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(deps.createSkill).toHaveBeenCalled(); + expect(deps.grantPermission).not.toHaveBeenCalled(); + expect(deps.upsertSkillFile).not.toHaveBeenCalled(); + expect(deps.deleteSkill).toHaveBeenCalledWith(expect.any(String)); + }); + + it('stops syncing after losing the Mongo lock lease', async () => { + jest.useFakeTimers(); + let releaseToken: (token: string) => void = () => undefined; + const tokenPromise = new Promise((resolve) => { + releaseToken = resolve; + }); + const deps = createDeps({ + getCredentialToken: jest.fn(() => tokenPromise), + refreshLock: jest.fn(async () => false), + }); + const runner = createGitHubSkillSyncRunner(deps); + + try { + const runPromise = runner.runOnce(); + await jest.advanceTimersByTimeAsync(10 * 60 * 1000); + releaseToken('github_pat_secret'); + const result = await runPromise; + + expect(result.status).toBe('failed'); + expect(result.message).toBe('GitHub skill sync lock was lost'); + expect(deps.fetchFn).not.toHaveBeenCalled(); + expect(deps.upsertStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: 'failed', + errorCode: 'SYNC_LOCK_LOST', + }), + ); + } finally { + jest.useRealTimers(); + } + }); +}); diff --git a/packages/api/src/skills/sync/github.ts b/packages/api/src/skills/sync/github.ts new file mode 100644 index 0000000000..5af41921f3 --- /dev/null +++ b/packages/api/src/skills/sync/github.ts @@ -0,0 +1,1872 @@ +import path from 'path'; +import crypto from 'crypto'; +import { Types } from 'mongoose'; +import { logger, tenantStorage } from '@librechat/data-schemas'; +import { + ResourceType, + PrincipalType, + AccessRoleIds, + SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH, +} from 'librechat-data-provider'; +import type { + ISkill, + ISkillFile, + CreateSkillInput, + UpdateSkillInput, + CreateSkillResult, + UpdateSkillResult, + UpsertSkillFileInput, + ISkillSyncStatus, + SkillSyncProvider, + SkillSyncCredentialSummary, + SkillSyncStatusInput, +} from '@librechat/data-schemas'; +import type { SkillSyncConfig, SkillSyncGitHubSourceConfig } from 'librechat-data-provider'; +import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits'; +import { parseSkillMarkdown } from '../parse'; + +const GITHUB_API_BASE = 'https://api.github.com'; +const SYSTEM_AUTHOR_ID = new Types.ObjectId('000000000000000000000000'); +const SYSTEM_AUTHOR_NAME = 'GitHub Sync'; +const PROVIDER: SkillSyncProvider = 'github'; +const LOCK_LEASE_MS = 30 * 60 * 1000; + +export const GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION = + 'Use a GitHub fine-grained personal access token scoped to the selected repository with read-only Contents and Metadata permissions.'; + +type FetchFn = typeof fetch; + +type GitHubTreeEntry = { + path: string; + mode: string; + type: 'blob' | 'tree' | 'commit'; + sha: string; + size?: number; + url: string; +}; + +type GitHubTreeResponse = { + sha: string; + tree: GitHubTreeEntry[]; + truncated: boolean; +}; + +type GitHubBlobResponse = { + sha: string; + content: string; + encoding: string; + size: number; +}; + +type GitHubCommitResponse = { + sha: string; + commit: { + tree: { + sha: string; + }; + }; +}; + +type SyncCounters = { + syncedSkillCount: number; + syncedFileCount: number; + deletedSkillCount: number; + deletedFileCount: number; +}; + +type AssertNotCancelled = () => void; + +type DiscoveredSkill = { + rootPath: string; + skillMd: GitHubTreeEntry; + files: GitHubTreeEntry[]; +}; + +type UpsertRemoteSkillResult = { + skill: ISkill & { _id: Types.ObjectId }; + created: boolean; +}; + +type PreparedRemoteSkill = { + existing: (ISkill & { _id: Types.ObjectId }) | null; + update: UpdateSkillInput; + createInput: CreateSkillInput; +}; + +type PreparedExistingRemoteSkill = PreparedRemoteSkill & { + existing: ISkill & { _id: Types.ObjectId }; +}; + +type PreparedDiscoveredSkill = { + discovered: DiscoveredSkill; + prepared: PreparedRemoteSkill; +}; + +type SaveBufferResult = { + filepath: string; + source: string; + storageKey?: string; + storageRegion?: string; +}; + +type StoredSkillFileRef = { + filepath: string; + source: string; + storageKey?: string; + storageRegion?: string; + author?: Types.ObjectId | string; + tenantId?: string; +}; + +type DeletedSyncedSkillJournal = { + skill: ISkill & { _id: Types.ObjectId }; + files: Array; +}; + +type SyncSkillFilesJournal = { + staleFiles: StoredSkillFileRef[]; + savedFiles: StoredSkillFileRef[]; +}; + +type SyncSkillFilesResult = Pick & + SyncSkillFilesJournal; + +type MaybePromise = T | Promise; + +export type GitHubSkillSyncDeps = { + getConfig: () => MaybePromise; + getCredentialToken: ( + provider: SkillSyncProvider, + credentialKey: string, + ) => Promise; + getCredentialSummary: ( + provider: SkillSyncProvider, + credentialKey: string, + ) => Promise; + listCredentials: (provider: SkillSyncProvider) => Promise; + listStatuses: (provider: SkillSyncProvider) => Promise; + upsertStatus: (input: SkillSyncStatusInput) => Promise; + tryAcquireLock: (params: { + provider: SkillSyncProvider; + lockOwner: string; + leaseMs: number; + tenantId?: string; + }) => Promise; + refreshLock: (params: { + provider: SkillSyncProvider; + lockOwner: string; + leaseMs: number; + tenantId?: string; + }) => Promise; + releaseLock: (params: { + provider: SkillSyncProvider; + lockOwner: string; + tenantId?: string; + }) => Promise; + createSkill: (data: CreateSkillInput) => Promise; + updateSkill: (params: { + id: string; + expectedVersion: number; + update: UpdateSkillInput; + }) => Promise; + getSkillById: (id: string | Types.ObjectId) => Promise<(ISkill & { _id: Types.ObjectId }) | null>; + findSkillBySourceIdentity: (params: { + source: 'github' | 'notion'; + upstreamId: string; + tenantId?: string; + }) => Promise<(ISkill & { _id: Types.ObjectId }) | null>; + listSkillsBySource: (params: { + source: 'github' | 'notion'; + sourceId: string; + }) => Promise>; + listSkillFiles: ( + skillId: string | Types.ObjectId, + ) => Promise>; + getSkillFileByPath: ( + skillId: string | Types.ObjectId, + relativePath: string, + ) => Promise<(ISkillFile & { _id: Types.ObjectId }) | null>; + upsertSkillFile: (row: UpsertSkillFileInput) => Promise; + deleteSkillFile: ( + skillId: string | Types.ObjectId, + relativePath: string, + ) => Promise<{ deleted: boolean }>; + deleteSkill: (id: string) => Promise<{ deleted: boolean }>; + saveBuffer: (params: { + userId: string; + buffer: Buffer; + fileName: string; + basePath?: string; + isImage?: boolean; + tenantId?: string; + }) => Promise; + deleteFile?: (file: { + filepath: string; + source: string; + storageKey?: string; + storageRegion?: string; + user?: Types.ObjectId | string; + tenantId?: string; + }) => Promise; + grantPermission: (params: { + principalType: string; + principalId: string | Types.ObjectId | null; + resourceType: string; + resourceId: string | Types.ObjectId; + accessRoleId: string; + grantedBy: string | Types.ObjectId; + }) => Promise; + fetchFn?: FetchFn; + lockOwner?: string; + allowServerCredentials?: boolean; +}; + +export type GitHubSkillSyncRunResult = { + status: 'started' | 'skipped' | 'completed' | 'failed'; + message?: string; + sources: Array; +}; + +export type GitHubSkillSyncStatus = { + enabled: boolean; + intervalMinutes: number; + runOnStartup: boolean; + sources: Array; + credentials: SkillSyncCredentialSummary[]; + fineGrainedTokenRecommendation: string; +}; + +export type GitHubSkillSyncRunner = { + getStatus: () => Promise; + runOnce: () => Promise; +}; + +class SkillSyncError extends Error { + code: string; + + constructor(code: string, message: string) { + super(message); + this.name = 'SkillSyncError'; + this.code = code; + } +} + +function normalizeRepoPath(value: string): string { + const trimmed = value.trim().replace(/^\/+|\/+$/g, ''); + return trimmed === '.' ? '' : trimmed; +} + +function isSafeRelativePath(value: string): boolean { + if (!value || value.startsWith('/') || value.startsWith('\\')) { + return false; + } + if (!/^[a-zA-Z0-9._\-/]+$/.test(value)) { + return false; + } + return value.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..'); +} + +function makeUpstreamId(source: SkillSyncGitHubSourceConfig, rootPath: string): string { + // Identity is keyed on the stable, admin-controlled source id and the skill's + // root path only — never owner/repo/ref. Repointing a source to a renamed or + // replacement repository (or rotating its ref) keeps the same upstream id, so + // existing mirrors are updated in place instead of being treated as new and + // colliding on the (name, author, tenantId) uniqueness constraint. + return `${source.id}:${rootPath}`; +} + +function makeSourceAuthorId(source: SkillSyncGitHubSourceConfig): Types.ObjectId { + // Fold the tenant into the synthetic author so the same source mirrored into + // different tenants gets distinct author ids (clearer audits, no cross-tenant + // author collisions). The tenant suffix is omitted when absent so single-tenant + // author ids stay stable. + const seed = source.tenantId + ? `${PROVIDER}:${source.id}:${source.tenantId}` + : `${PROVIDER}:${source.id}`; + const digest = crypto.createHash('sha256').update(seed).digest('hex').slice(0, 24); + return new Types.ObjectId(digest); +} + +function toSkillName(value: string): string { + const normalized = value + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/^-+|-+$/g, '') + .replace(/-{2,}/g, '-'); + return normalized || 'github-skill'; +} + +function getFilename(relativePath: string): string { + return path.posix.basename(relativePath); +} + +function guessMimeType(filename: string): string { + const ext = path.extname(filename).toLowerCase(); + const mimeMap: Record = { + '.md': 'text/markdown', + '.txt': 'text/plain', + '.js': 'application/javascript', + '.ts': 'text/typescript', + '.jsx': 'text/jsx', + '.tsx': 'text/tsx', + '.json': 'application/json', + '.yaml': 'text/yaml', + '.yml': 'text/yaml', + '.py': 'text/x-python', + '.sh': 'application/x-sh', + '.css': 'text/css', + '.html': 'text/html', + '.xml': 'application/xml', + '.csv': 'text/csv', + '.toml': 'text/toml', + '.ini': 'text/ini', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.svg': 'image/svg+xml', + '.webp': 'image/webp', + '.pdf': 'application/pdf', + }; + return mimeMap[ext] ?? 'application/octet-stream'; +} + +function toCleanFrontmatter( + frontmatter: Record | undefined, +): Record { + if (!frontmatter) { + return {}; + } + const clean = { ...frontmatter }; + delete clean.name; + delete clean.description; + // Drop a placeholder/non-boolean always-apply (e.g. `always-apply:` or + // `always-apply: # TODO`, which js-yaml yields as null). Apply the same + // cleanup to the accepted `alwaysApply` alias so a malformed alias does not + // survive after the canonical key has already supplied the effective flag. + // The boolean is already captured in the dedicated alwaysApply field, and + // persisting a null here would leave ambiguous/invalid frontmatter on the + // synced skill. + if ('always-apply' in clean && typeof clean['always-apply'] !== 'boolean') { + delete clean['always-apply']; + } + if ('alwaysApply' in clean && typeof clean.alwaysApply !== 'boolean') { + delete clean.alwaysApply; + } + return clean; +} + +function getLimitMegabytes(bytes: number): number { + return Math.round(bytes / 1024 / 1024); +} + +function assertGitHubBlobSize(entry: GitHubTreeEntry, relativePath: string): number { + if (typeof entry.size !== 'number' || !Number.isFinite(entry.size) || entry.size < 0) { + throw new SkillSyncError( + 'GITHUB_BLOB_SIZE_UNKNOWN', + `GitHub file "${relativePath}" did not include a valid blob size`, + ); + } + if (entry.size > DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes) { + throw new SkillSyncError( + 'GITHUB_BLOB_TOO_LARGE', + `GitHub file "${relativePath}" exceeds the ${getLimitMegabytes( + DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes, + )}MB per-file skill import limit`, + ); + } + return entry.size; +} + +function assertGitHubBufferSize(buffer: Buffer, relativePath: string): void { + if (buffer.length <= DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes) { + return; + } + throw new SkillSyncError( + 'GITHUB_BLOB_TOO_LARGE', + `GitHub file "${relativePath}" exceeds the ${getLimitMegabytes( + DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes, + )}MB per-file skill import limit`, + ); +} + +function assertCumulativeGitHubFileSize(totalBytes: number): void { + if (totalBytes <= DEFAULT_SKILL_IMPORT_LIMITS.maxDecompressedBytes) { + return; + } + throw new SkillSyncError( + 'GITHUB_PACKAGE_TOO_LARGE', + `GitHub skill files exceed the ${getLimitMegabytes( + DEFAULT_SKILL_IMPORT_LIMITS.maxDecompressedBytes, + )}MB cumulative skill import limit`, + ); +} + +function assertGitHubEntryCount(discovered: DiscoveredSkill): void { + const entryCount = discovered.files.length + 1; + if (entryCount <= DEFAULT_SKILL_IMPORT_LIMITS.maxEntries) { + return; + } + throw new SkillSyncError( + 'GITHUB_TOO_MANY_FILES', + `GitHub skill "${discovered.rootPath}" exceeds the ${DEFAULT_SKILL_IMPORT_LIMITS.maxEntries} file skill import limit`, + ); +} + +function getSkillMdPath(discovered: DiscoveredSkill): string { + return discovered.rootPath ? `${discovered.rootPath}/SKILL.md` : 'SKILL.md'; +} + +function getDiscoveredRelativePath(discovered: DiscoveredSkill, entry: GitHubTreeEntry): string { + const prefix = discovered.rootPath ? `${discovered.rootPath}/` : ''; + const normalized = normalizeRepoPath(entry.path); + return prefix ? normalized.slice(prefix.length) : normalized; +} + +function assertGitHubSkillPackageManifest(discovered: DiscoveredSkill): void { + assertGitHubEntryCount(discovered); + assertGitHubBlobSize(discovered.skillMd, getSkillMdPath(discovered)); + let totalFileBytes = 0; + for (const entry of discovered.files) { + const relativePath = getDiscoveredRelativePath(discovered, entry); + if (!isSafeRelativePath(relativePath) || relativePath.toUpperCase() === 'SKILL.MD') { + continue; + } + totalFileBytes += assertGitHubBlobSize(entry, relativePath); + assertCumulativeGitHubFileSize(totalFileBytes); + } +} + +function getSourceMetadataString( + row: { sourceMetadata?: Record }, + key: string, +): string | undefined { + const metadata = row.sourceMetadata; + const value = metadata && typeof metadata === 'object' ? metadata[key] : undefined; + return typeof value === 'string' ? value : undefined; +} + +function serializeDate(date: Date): string { + return date.toISOString(); +} + +function sanitizeError(error: unknown): { code: string; message: string } { + if (error instanceof SkillSyncError) { + return { code: error.code, message: error.message }; + } + if (error instanceof Error) { + return { + code: 'SYNC_FAILED', + message: error.message.replace(/Bearer\s+\S+/gi, 'Bearer [redacted]'), + }; + } + return { code: 'SYNC_FAILED', message: 'Unknown skill sync failure' }; +} + +function buildGitHubHeaders(token: string): HeadersInit { + return { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'LibreChat-Skill-Sync', + }; +} + +function buildGitHubUrl(pathname: string): string { + return `${GITHUB_API_BASE}${pathname}`; +} + +function encodeGitHubPath(value: string): string { + return value.split('/').map(encodeURIComponent).join('/'); +} + +async function readGitHubErrorMessage(response: Response): Promise { + try { + const body = (await response.json()) as { message?: unknown }; + return typeof body.message === 'string' ? body.message : undefined; + } catch { + return undefined; + } +} + +function isGitHubRateLimitResponse(params: { + status: number; + remaining: string | null; + retryAfter: string | null; + message?: string; +}): boolean { + if (params.status === 429 || params.remaining === '0' || params.retryAfter) { + return true; + } + const message = params.message?.toLowerCase() ?? ''; + return message.includes('rate limit') || message.includes('abuse detection'); +} + +async function githubJson(params: { + fetchFn: FetchFn; + token: string; + pathname: string; +}): Promise { + const response = await params.fetchFn(buildGitHubUrl(params.pathname), { + headers: buildGitHubHeaders(params.token), + }); + if (response.ok) { + return (await response.json()) as T; + } + const remaining = response.headers.get('x-ratelimit-remaining'); + const retryAfter = response.headers.get('retry-after'); + const message = await readGitHubErrorMessage(response); + if (response.status === 401 || response.status === 403 || response.status === 429) { + const code = isGitHubRateLimitResponse({ + status: response.status, + remaining, + retryAfter, + message, + }) + ? 'GITHUB_RATE_LIMITED' + : 'GITHUB_AUTH_FAILED'; + throw new SkillSyncError(code, `GitHub request failed with HTTP ${response.status}`); + } + if (response.status === 404) { + throw new SkillSyncError('GITHUB_NOT_FOUND', 'GitHub repository, ref, or path was not found'); + } + throw new SkillSyncError( + 'GITHUB_REQUEST_FAILED', + `GitHub request failed with HTTP ${response.status}`, + ); +} + +async function fetchCommit(params: { + fetchFn: FetchFn; + token: string; + source: SkillSyncGitHubSourceConfig; +}): Promise { + const owner = encodeURIComponent(params.source.owner); + const repo = encodeURIComponent(params.source.repo); + const ref = encodeGitHubPath(params.source.ref); + return githubJson({ + fetchFn: params.fetchFn, + token: params.token, + pathname: `/repos/${owner}/${repo}/commits/${ref}`, + }); +} + +async function fetchTree(params: { + fetchFn: FetchFn; + token: string; + source: SkillSyncGitHubSourceConfig; + treeSha: string; + recursive?: boolean; +}): Promise { + const owner = encodeURIComponent(params.source.owner); + const repo = encodeURIComponent(params.source.repo); + const treeSha = encodeURIComponent(params.treeSha); + const recursive = params.recursive ?? true; + return githubJson({ + fetchFn: params.fetchFn, + token: params.token, + pathname: `/repos/${owner}/${repo}/git/trees/${treeSha}${recursive ? '?recursive=1' : ''}`, + }); +} + +async function fetchTreeAtPath(params: { + fetchFn: FetchFn; + token: string; + source: SkillSyncGitHubSourceConfig; + rootTreeSha: string; + repoPath: string; + assertNotCancelled: AssertNotCancelled; +}): Promise { + const normalizedPath = normalizeRepoPath(params.repoPath); + let treeSha = params.rootTreeSha; + if (normalizedPath) { + for (const segment of normalizedPath.split('/')) { + params.assertNotCancelled(); + const tree = await fetchTree({ + fetchFn: params.fetchFn, + token: params.token, + source: params.source, + treeSha, + recursive: false, + }); + params.assertNotCancelled(); + if (tree.truncated) { + throw new SkillSyncError('GITHUB_TREE_TRUNCATED', 'GitHub tree response was truncated'); + } + const next = tree.tree.find((entry) => entry.type === 'tree' && entry.path === segment); + if (!next) { + throw new SkillSyncError( + 'GITHUB_PATH_NOT_FOUND', + `Configured GitHub skill path "${normalizedPath}" was not found`, + ); + } + treeSha = next.sha; + } + } + + params.assertNotCancelled(); + const tree = await fetchTree({ + fetchFn: params.fetchFn, + token: params.token, + source: params.source, + treeSha, + recursive: true, + }); + params.assertNotCancelled(); + if (tree.truncated) { + throw new SkillSyncError('GITHUB_TREE_TRUNCATED', 'GitHub tree response was truncated'); + } + if (!normalizedPath) { + return tree.tree; + } + return tree.tree.map((entry) => ({ + ...entry, + path: `${normalizedPath}/${normalizeRepoPath(entry.path)}`, + })); +} + +async function fetchConfiguredTreeEntries(params: { + fetchFn: FetchFn; + token: string; + source: SkillSyncGitHubSourceConfig; + rootTreeSha: string; + assertNotCancelled: AssertNotCancelled; +}): Promise { + const entriesByPath = new Map(); + for (const repoPath of params.source.paths) { + const entries = await fetchTreeAtPath({ ...params, repoPath }); + for (const entry of entries) { + const normalizedPath = normalizeRepoPath(entry.path); + entriesByPath.set(normalizedPath, { ...entry, path: normalizedPath }); + } + } + return [...entriesByPath.values()]; +} + +async function fetchBlob(params: { + fetchFn: FetchFn; + token: string; + source: SkillSyncGitHubSourceConfig; + sha: string; +}): Promise { + const owner = encodeURIComponent(params.source.owner); + const repo = encodeURIComponent(params.source.repo); + const sha = encodeURIComponent(params.sha); + const blob = await githubJson({ + fetchFn: params.fetchFn, + token: params.token, + pathname: `/repos/${owner}/${repo}/git/blobs/${sha}`, + }); + if (blob.encoding !== 'base64') { + throw new SkillSyncError( + 'GITHUB_UNSUPPORTED_BLOB', + `Unsupported GitHub blob encoding "${blob.encoding}"`, + ); + } + return Buffer.from(blob.content.replace(/\s/g, ''), 'base64'); +} + +function isSkillRootWithinDiscoveryDepth( + rootPath: string, + basePath: string, + maxDepth: number, +): boolean { + if (rootPath === basePath) { + return true; + } + if (basePath && !rootPath.startsWith(`${basePath}/`)) { + return false; + } + const relative = basePath ? rootPath.slice(basePath.length).replace(/^\/+/, '') : rootPath; + if (!relative) { + return true; + } + return relative.split('/').length <= maxDepth; +} + +function discoverSkills( + tree: GitHubTreeEntry[], + source: SkillSyncGitHubSourceConfig, +): DiscoveredSkill[] { + const basePaths = source.paths.map(normalizeRepoPath); + const skillDiscoveryDepth = source.skillDiscoveryDepth ?? SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH; + const skillMdByRoot = new Map(); + for (const entry of tree) { + if (entry.type !== 'blob') { + continue; + } + const normalized = normalizeRepoPath(entry.path); + const basename = path.posix.basename(normalized); + if (basename.toUpperCase() !== 'SKILL.MD') { + continue; + } + const parent = normalizeRepoPath(path.posix.dirname(normalized)); + for (const basePath of basePaths) { + if (isSkillRootWithinDiscoveryDepth(parent, basePath, skillDiscoveryDepth)) { + skillMdByRoot.set(parent, entry); + } + } + } + + const skillRoots = [...skillMdByRoot.keys()]; + return [...skillMdByRoot.entries()].map(([rootPath, skillMd]) => { + const prefix = rootPath ? `${rootPath}/` : ''; + const childSkillRoots = skillRoots.filter((candidate) => { + if (!candidate || candidate === rootPath) { + return false; + } + return rootPath ? candidate.startsWith(`${rootPath}/`) : true; + }); + const files = tree.filter((entry) => { + if (entry.type !== 'blob') { + return false; + } + const normalized = normalizeRepoPath(entry.path); + if (!normalized.startsWith(prefix) || normalized === skillMd.path) { + return false; + } + if (childSkillRoots.some((childRoot) => normalized.startsWith(`${childRoot}/`))) { + return false; + } + const relativePath = prefix ? normalized.slice(prefix.length) : normalized; + return isSafeRelativePath(relativePath) && relativePath.toUpperCase() !== 'SKILL.MD'; + }); + return { rootPath, skillMd, files }; + }); +} + +function assertConfiguredPathsExist( + tree: GitHubTreeEntry[], + source: SkillSyncGitHubSourceConfig, +): void { + for (const configuredPath of source.paths.map(normalizeRepoPath)) { + if (configuredPath === '') { + continue; + } + const exists = tree.some((entry) => { + const entryPath = normalizeRepoPath(entry.path); + return entryPath === configuredPath || entryPath.startsWith(`${configuredPath}/`); + }); + if (!exists) { + throw new SkillSyncError( + 'GITHUB_PATH_NOT_FOUND', + `Configured GitHub skill path "${configuredPath}" was not found`, + ); + } + } +} + +function makeStatusInput(params: { + source: SkillSyncGitHubSourceConfig; + status: SkillSyncStatusInput['status']; + startedAt?: Date; + finishedAt?: Date; + errorCode?: string; + errorMessage?: string; + counts?: Partial; +}): SkillSyncStatusInput { + return { + provider: PROVIDER, + sourceId: params.source.id, + tenantId: params.source.tenantId, + status: params.status, + credentialKey: params.source.credentialKey, + owner: params.source.owner, + repo: params.source.repo, + ref: params.source.ref, + paths: params.source.paths, + startedAt: params.startedAt, + finishedAt: params.finishedAt, + errorCode: params.errorCode, + errorMessage: params.errorMessage, + syncedSkillCount: params.counts?.syncedSkillCount ?? 0, + syncedFileCount: params.counts?.syncedFileCount ?? 0, + deletedSkillCount: params.counts?.deletedSkillCount ?? 0, + deletedFileCount: params.counts?.deletedFileCount ?? 0, + }; +} + +function makeStatusKey(sourceId: string, tenantId?: string): string { + return `${tenantId ?? ''}:${sourceId}`; +} + +async function ensurePublicViewer( + deps: GitHubSkillSyncDeps, + skillId: Types.ObjectId, +): Promise { + await deps.grantPermission({ + principalType: PrincipalType.PUBLIC, + principalId: null, + resourceType: ResourceType.SKILL, + resourceId: skillId, + accessRoleId: AccessRoleIds.SKILL_VIEWER, + grantedBy: SYSTEM_AUTHOR_ID, + }); +} + +async function prepareRemoteSkill(params: { + deps: GitHubSkillSyncDeps; + source: SkillSyncGitHubSourceConfig; + discovered: DiscoveredSkill; + skillMdContent: string; + commitSha: string; + syncedAt: Date; +}): Promise { + const { deps, source, discovered, skillMdContent, commitSha, syncedAt } = params; + const parsed = parseSkillMarkdown(skillMdContent); + if (parsed.parseError) { + throw new SkillSyncError( + 'SKILL_PARSE_FAILED', + `${discovered.rootPath}/SKILL.md contains invalid YAML frontmatter: ${parsed.parseError}`, + ); + } + if (parsed.invalidBooleans.length > 0) { + throw new SkillSyncError( + 'SKILL_PARSE_FAILED', + `${discovered.rootPath}/SKILL.md contains invalid boolean frontmatter`, + ); + } + const upstreamId = makeUpstreamId(source, discovered.rootPath); + const fallbackName = toSkillName(path.posix.basename(discovered.rootPath) || source.id); + const sourceMetadata = { + provider: PROVIDER, + sourceId: source.id, + upstreamId, + owner: source.owner, + repo: source.repo, + ref: source.ref, + skillPath: discovered.rootPath, + commitSha, + skillBlobSha: discovered.skillMd.sha, + syncedAt: serializeDate(syncedAt), + syncStatus: 'synced', + }; + const update: UpdateSkillInput = { + name: parsed.name || fallbackName, + description: parsed.description || parsed.name || fallbackName, + body: skillMdContent, + frontmatter: toCleanFrontmatter(parsed.frontmatter), + alwaysApply: parsed.alwaysApply, + source: PROVIDER, + sourceMetadata, + }; + const sourceTenantId = source.tenantId ?? undefined; + const foundExisting = await deps.findSkillBySourceIdentity({ + source: PROVIDER, + upstreamId, + tenantId: sourceTenantId, + }); + const existing = + foundExisting && (foundExisting.tenantId ?? undefined) === sourceTenantId + ? foundExisting + : null; + const createInput: CreateSkillInput = { + ...(update as Omit), + name: update.name ?? fallbackName, + description: update.description ?? fallbackName, + author: makeSourceAuthorId(source), + authorName: SYSTEM_AUTHOR_NAME, + source: PROVIDER, + tenantId: source.tenantId, + }; + return { existing, update, createInput }; +} + +async function commitRemoteSkill( + deps: GitHubSkillSyncDeps, + prepared: PreparedRemoteSkill, +): Promise { + if (prepared.existing) { + const result = await deps.updateSkill({ + id: prepared.existing._id.toString(), + expectedVersion: prepared.existing.version, + update: prepared.update, + }); + if (result.status === 'updated') { + return { skill: result.skill, created: false }; + } + if (result.status === 'conflict') { + throw new SkillSyncError( + 'SKILL_CONFLICT', + `Skill "${prepared.existing.name}" changed during sync`, + ); + } + throw new SkillSyncError( + 'SKILL_NOT_FOUND', + `Previously synced skill "${prepared.existing.name}" was removed`, + ); + } + const created = await deps.createSkill(prepared.createInput); + return { skill: created.skill, created: true }; +} + +/** + * File sync bumps the parent skill's `version` (via file upserts/deletes) but + * never changes its authored content, so we must re-read to get past our own + * version bumps. A plain re-read would also silently accept and overwrite a + * concurrent external edit; compare the refreshed content against the pre-sync + * snapshot and treat a changed body/name/description/always-apply as a conflict. + */ +function hasExternalSkillEdit(before: ISkill, after: ISkill): boolean { + return ( + before.body !== after.body || + before.name !== after.name || + before.description !== after.description || + (before.alwaysApply ?? false) !== (after.alwaysApply ?? false) || + JSON.stringify(before.frontmatter ?? {}) !== JSON.stringify(after.frontmatter ?? {}) + ); +} + +async function commitExistingRemoteSkillAfterFileSync( + deps: GitHubSkillSyncDeps, + prepared: PreparedExistingRemoteSkill, + options: { forceCommit?: boolean } = {}, +): Promise { + const refreshed = await deps.getSkillById(prepared.existing._id); + if (!refreshed) { + throw new SkillSyncError( + 'SKILL_NOT_FOUND', + `Previously synced skill "${prepared.existing.name}" was removed`, + ); + } + if (hasExternalSkillEdit(prepared.existing, refreshed)) { + throw new SkillSyncError( + 'SKILL_CONFLICT', + `Skill "${prepared.existing.name}" was modified during sync`, + ); + } + if (!options.forceCommit && !hasRemoteSkillDefinitionChanged(prepared.update, refreshed)) { + return { skill: refreshed, created: false }; + } + return commitRemoteSkill(deps, { ...prepared, existing: refreshed }); +} + +async function cleanupFile(deps: GitHubSkillSyncDeps, file: StoredSkillFileRef): Promise { + if (!deps.deleteFile) { + return; + } + await deps.deleteFile({ + filepath: file.filepath, + source: file.source, + storageKey: file.storageKey, + storageRegion: file.storageRegion, + user: file.author, + tenantId: file.tenantId, + }); +} + +function toStoredFileRef(params: { + saved: SaveBufferResult; + author: Types.ObjectId; + tenantId?: string; +}): StoredSkillFileRef { + return { + filepath: params.saved.filepath, + source: params.saved.source, + storageKey: params.saved.storageKey, + storageRegion: params.saved.storageRegion, + author: params.author, + tenantId: params.tenantId, + }; +} + +function toSkillFileInput(file: ISkillFile & { _id: Types.ObjectId }): UpsertSkillFileInput { + return { + skillId: file.skillId, + relativePath: file.relativePath, + file_id: file.file_id, + filename: file.filename, + filepath: file.filepath, + storageKey: file.storageKey, + storageRegion: file.storageRegion, + source: file.source, + sourceMetadata: file.sourceMetadata, + mimeType: file.mimeType, + bytes: file.bytes, + isExecutable: file.isExecutable, + author: file.author, + tenantId: file.tenantId, + }; +} + +function toCreateSkillInput(skill: ISkill & { _id: Types.ObjectId }): CreateSkillInput { + return { + name: skill.name, + displayTitle: skill.displayTitle, + description: skill.description, + body: skill.body, + frontmatter: skill.frontmatter, + category: skill.category, + author: skill.author, + authorName: skill.authorName, + source: PROVIDER, + sourceMetadata: skill.sourceMetadata, + alwaysApply: skill.alwaysApply, + tenantId: skill.tenantId, + }; +} + +function toStoredFileRefFromSkillFile( + file: ISkillFile & { _id: Types.ObjectId }, +): StoredSkillFileRef { + return { + filepath: file.filepath, + source: file.source, + storageKey: file.storageKey, + storageRegion: file.storageRegion, + author: file.author, + tenantId: file.tenantId, + }; +} + +function getStoredFileKey(file: StoredSkillFileRef): string { + return [file.source, file.filepath, file.storageKey ?? '', file.storageRegion ?? ''].join(':'); +} + +async function cleanupStoredFiles(params: { + deps: GitHubSkillSyncDeps; + files: StoredSkillFileRef[]; + logMessage: string; +}): Promise { + const seen = new Set(); + for (const file of params.files) { + const key = getStoredFileKey(file); + if (seen.has(key)) { + continue; + } + seen.add(key); + await cleanupFile(params.deps, file).catch((cleanupError) => + logger.error(params.logMessage, cleanupError), + ); + } +} + +async function restoreExistingSkillFiles(params: { + deps: GitHubSkillSyncDeps; + skill: ISkill & { _id: Types.ObjectId }; + previousFiles: Array; + savedFiles: StoredSkillFileRef[]; +}): Promise { + const { deps, skill, previousFiles, savedFiles } = params; + const previousByPath = new Map(previousFiles.map((file) => [file.relativePath, file])); + const currentFiles = await deps.listSkillFiles(skill._id); + + for (const file of currentFiles) { + if (previousByPath.has(file.relativePath)) { + continue; + } + await deps.deleteSkillFile(skill._id, file.relativePath); + } + for (const file of previousFiles) { + await deps.upsertSkillFile(toSkillFileInput(file)); + } + await cleanupStoredFiles({ + deps, + files: savedFiles, + logMessage: '[GitHubSkillSync] Failed to clean up rolled-back synced file:', + }); +} + +async function deleteSyncedSkillForRestore( + deps: GitHubSkillSyncDeps, + skill: ISkill & { _id: Types.ObjectId }, +): Promise<{ deletedFileCount: number; deletedSkill: DeletedSyncedSkillJournal }> { + const files = await deps.listSkillFiles(skill._id); + await deps.deleteSkill(skill._id.toString()); + return { + deletedFileCount: files.length, + deletedSkill: { skill, files }, + }; +} + +async function restoreDeletedSyncedSkill( + deps: GitHubSkillSyncDeps, + deleted: DeletedSyncedSkillJournal, +): Promise { + const restored = await deps.createSkill(toCreateSkillInput(deleted.skill)); + for (const file of deleted.files) { + await deps.upsertSkillFile({ + ...toSkillFileInput(file), + skillId: restored.skill._id, + }); + } + await ensurePublicViewer(deps, restored.skill._id); +} + +async function cleanupDeletedSyncedSkillFiles( + deps: GitHubSkillSyncDeps, + deleted: DeletedSyncedSkillJournal, +): Promise { + await cleanupStoredFiles({ + deps, + files: deleted.files.map(toStoredFileRefFromSkillFile), + logMessage: '[GitHubSkillSync] Failed to clean up deleted stale mirrored skill file:', + }); +} + +function comparableSourceMetadata(metadata: Record | undefined): string { + const { commitSha: _commitSha, syncedAt: _syncedAt, ...rest } = metadata ?? {}; + return JSON.stringify(rest); +} + +function hasRemoteSkillDefinitionChanged(update: UpdateSkillInput, existing: ISkill): boolean { + return ( + update.body !== existing.body || + update.name !== existing.name || + update.description !== existing.description || + (update.alwaysApply ?? false) !== (existing.alwaysApply ?? false) || + JSON.stringify(update.frontmatter ?? {}) !== JSON.stringify(existing.frontmatter ?? {}) || + comparableSourceMetadata(update.sourceMetadata) !== + comparableSourceMetadata(existing.sourceMetadata) + ); +} + +function findMovedSourceSkill(params: { + source: SkillSyncGitHubSourceConfig; + prepared: PreparedRemoteSkill; + existingSyncedSkills: Array; + excludedUpstreamIds: Set; +}): (ISkill & { _id: Types.ObjectId }) | null { + const sourceTenantId = params.source.tenantId ?? undefined; + const sourceAuthor = params.prepared.createInput.author.toString(); + const name = params.prepared.createInput.name; + + return ( + params.existingSyncedSkills.find((skill) => { + if ((skill.tenantId ?? undefined) !== sourceTenantId) { + return false; + } + if (skill.name !== name || skill.author.toString() !== sourceAuthor) { + return false; + } + const upstreamId = getSourceMetadataString(skill, 'upstreamId'); + if (!upstreamId) { + return false; + } + return !params.excludedUpstreamIds.has(upstreamId); + }) ?? null + ); +} + +function hasNameConflictingStaleSkill(params: { + source: SkillSyncGitHubSourceConfig; + prepared: PreparedDiscoveredSkill; + existingSyncedSkills: Array; + discoveredUpstreamIds: Set; +}): boolean { + return Boolean( + findMovedSourceSkill({ + source: params.source, + prepared: params.prepared.prepared, + existingSyncedSkills: params.existingSyncedSkills, + excludedUpstreamIds: params.discoveredUpstreamIds, + }), + ); +} + +function orderPreparedSkillsForSafeStaleDeletes(params: { + source: SkillSyncGitHubSourceConfig; + preparedSkills: PreparedDiscoveredSkill[]; + existingSyncedSkills: Array; + discoveredUpstreamIds: Set; +}): PreparedDiscoveredSkill[] { + const regular: PreparedDiscoveredSkill[] = []; + const nameConflicting: PreparedDiscoveredSkill[] = []; + for (const prepared of params.preparedSkills) { + if ( + prepared.prepared.existing && + hasNameConflictingStaleSkill({ + source: params.source, + prepared, + existingSyncedSkills: params.existingSyncedSkills, + discoveredUpstreamIds: params.discoveredUpstreamIds, + }) + ) { + nameConflicting.push(prepared); + continue; + } + regular.push(prepared); + } + return [...regular, ...nameConflicting]; +} + +function getMirrorNameKey(params: { + tenantId?: string; + author: string; + name: string | undefined; +}): string { + return `${params.tenantId ?? ''}:${params.author}:${params.name ?? ''}`; +} + +function assertNoDuplicatePreparedSkillNames( + source: SkillSyncGitHubSourceConfig, + preparedSkills: PreparedDiscoveredSkill[], +): void { + const sourceTenantId = source.tenantId ?? undefined; + const seen = new Map(); + for (const { discovered, prepared } of preparedSkills) { + const key = getMirrorNameKey({ + tenantId: sourceTenantId, + author: prepared.createInput.author.toString(), + name: prepared.createInput.name, + }); + if (seen.has(key)) { + throw new SkillSyncError( + 'DUPLICATE_SKILL_NAME', + `GitHub source "${source.id}" contains multiple skills named "${prepared.createInput.name}"`, + ); + } + seen.set(key, discovered.rootPath); + } +} + +async function deleteNameConflictingStaleSkill(params: { + deps: GitHubSkillSyncDeps; + source: SkillSyncGitHubSourceConfig; + prepared: PreparedRemoteSkill; + existingSyncedSkills: Array; + discoveredUpstreamIds: Set; + assertNotCancelled: AssertNotCancelled; +}): Promise<{ + remainingSkills: Array; + deletedSkillCount: number; + deletedFileCount: number; + deletedSkill?: DeletedSyncedSkillJournal; +}> { + const staleSkill = findMovedSourceSkill({ + source: params.source, + prepared: params.prepared, + existingSyncedSkills: params.existingSyncedSkills, + excludedUpstreamIds: params.discoveredUpstreamIds, + }); + if (!staleSkill) { + return { + remainingSkills: params.existingSyncedSkills, + deletedSkillCount: 0, + deletedFileCount: 0, + }; + } + + params.assertNotCancelled(); + const { deletedFileCount, deletedSkill } = await deleteSyncedSkillForRestore( + params.deps, + staleSkill, + ); + const staleSkillId = staleSkill._id.toString(); + + return { + remainingSkills: params.existingSyncedSkills.filter( + (skill) => skill._id.toString() !== staleSkillId, + ), + deletedSkillCount: 1, + deletedFileCount, + deletedSkill, + }; +} + +async function syncSkillFiles(params: { + deps: GitHubSkillSyncDeps; + token: string; + source: SkillSyncGitHubSourceConfig; + skill: ISkill & { _id: Types.ObjectId }; + discovered: DiscoveredSkill; + commitSha: string; + fetchFn: FetchFn; + assertNotCancelled: AssertNotCancelled; + journal?: SyncSkillFilesJournal; +}): Promise { + const { deps, token, source, skill, discovered, commitSha, fetchFn, assertNotCancelled } = params; + const journal = params.journal ?? { staleFiles: [], savedFiles: [] }; + const remotePaths = new Set(); + let syncedFileCount = 0; + let deletedFileCount = 0; + let totalFileBytes = 0; + + for (const entry of discovered.files) { + assertNotCancelled(); + const relativePath = getDiscoveredRelativePath(discovered, entry); + if (!isSafeRelativePath(relativePath) || relativePath.toUpperCase() === 'SKILL.MD') { + continue; + } + totalFileBytes += assertGitHubBlobSize(entry, relativePath); + assertCumulativeGitHubFileSize(totalFileBytes); + remotePaths.add(relativePath); + const existing = await deps.getSkillFileByPath(skill._id, relativePath); + if (existing && getSourceMetadataString(existing, 'blobSha') === entry.sha) { + continue; + } + const buffer = await fetchBlob({ fetchFn, token, source, sha: entry.sha }); + assertNotCancelled(); + assertGitHubBufferSize(buffer, relativePath); + const fileId = crypto.randomUUID(); + const filename = getFilename(relativePath); + const mimeType = guessMimeType(filename); + const saved = await deps.saveBuffer({ + userId: skill.author.toString(), + buffer, + fileName: `${fileId}__${filename}`, + basePath: 'uploads', + isImage: mimeType.startsWith('image/'), + tenantId: skill.tenantId, + }); + const savedFile = toStoredFileRef({ saved, author: skill.author, tenantId: skill.tenantId }); + try { + await deps.upsertSkillFile({ + skillId: skill._id, + relativePath, + file_id: fileId, + filename, + filepath: saved.filepath, + storageKey: saved.storageKey, + storageRegion: saved.storageRegion, + source: saved.source, + sourceMetadata: { + provider: PROVIDER, + sourceId: source.id, + upstreamId: makeUpstreamId(source, discovered.rootPath), + commitSha, + blobSha: entry.sha, + path: entry.path, + }, + mimeType, + bytes: buffer.length, + isExecutable: false, + author: skill.author, + tenantId: skill.tenantId, + }); + } catch (error) { + await cleanupFile(deps, savedFile).catch((cleanupError) => + logger.error('[GitHubSkillSync] Failed to clean up orphaned synced file:', cleanupError), + ); + throw error; + } + syncedFileCount++; + journal.savedFiles.push(savedFile); + if (existing && existing.filepath !== saved.filepath) { + journal.staleFiles.push(existing); + } + } + + const existingFiles = await deps.listSkillFiles(skill._id); + for (const file of existingFiles) { + assertNotCancelled(); + if (remotePaths.has(file.relativePath)) { + continue; + } + const result = await deps.deleteSkillFile(skill._id, file.relativePath); + if (result.deleted) { + deletedFileCount++; + journal.staleFiles.push(file); + } + } + return { syncedFileCount, deletedFileCount, ...journal }; +} + +async function deleteSyncedSkill( + deps: GitHubSkillSyncDeps, + skill: ISkill & { _id: Types.ObjectId }, +): Promise { + const files = await deps.listSkillFiles(skill._id); + let deletedFiles = 0; + for (const file of files) { + await cleanupFile(deps, file).catch((cleanupError) => + logger.error('[GitHubSkillSync] Failed to clean up mirrored skill file:', cleanupError), + ); + deletedFiles++; + } + await deps.deleteSkill(skill._id.toString()); + return deletedFiles; +} + +function getTokenEnvVarName(tokenReference: string | undefined): string | null { + const match = tokenReference?.trim().match(/^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/); + return match?.[1] ?? null; +} + +async function resolveGitHubToken( + deps: GitHubSkillSyncDeps, + source: SkillSyncGitHubSourceConfig, +): Promise { + if (deps.allowServerCredentials === false) { + return null; + } + const tokenEnvVar = getTokenEnvVarName(source.token); + if (tokenEnvVar) { + return process.env[tokenEnvVar]?.trim() || null; + } + if (!source.credentialKey) { + return null; + } + return deps.getCredentialToken(PROVIDER, source.credentialKey); +} + +function getMissingCredentialMessage( + source: SkillSyncGitHubSourceConfig, + allowServerCredentials: boolean, +): string { + if (!allowServerCredentials) { + return 'Server GitHub credentials are not available for this skill sync config'; + } + const tokenEnvVar = getTokenEnvVarName(source.token); + if (tokenEnvVar) { + return `Missing GitHub token environment variable "${tokenEnvVar}"`; + } + return `Missing GitHub credential "${source.credentialKey ?? source.id}"`; +} + +async function syncSource(params: { + deps: GitHubSkillSyncDeps; + source: SkillSyncGitHubSourceConfig; + fetchFn: FetchFn; + assertNotCancelled: AssertNotCancelled; +}): Promise { + const { deps, source, fetchFn, assertNotCancelled } = params; + const startedAt = new Date(); + await deps.upsertStatus(makeStatusInput({ source, status: 'running', startedAt })); + try { + assertNotCancelled(); + const allowServerCredentials = deps.allowServerCredentials !== false; + const token = await resolveGitHubToken(deps, source); + assertNotCancelled(); + if (!token) { + throw new SkillSyncError( + 'MISSING_CREDENTIAL', + getMissingCredentialMessage(source, allowServerCredentials), + ); + } + const commit = await fetchCommit({ fetchFn, token, source }); + assertNotCancelled(); + const treeEntries = await fetchConfiguredTreeEntries({ + fetchFn, + token, + source, + rootTreeSha: commit.commit.tree.sha, + assertNotCancelled, + }); + assertConfiguredPathsExist(treeEntries, source); + const discoveredSkills = discoverSkills(treeEntries, source); + const seenUpstreamIds = new Set(); + let existingSyncedSkills: Array | null = null; + const getExistingSyncedSkills = async () => { + if (!existingSyncedSkills) { + existingSyncedSkills = await deps.listSkillsBySource({ + source: PROVIDER, + sourceId: source.id, + }); + } + return existingSyncedSkills; + }; + const counts: SyncCounters = { + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + }; + const syncedAt = new Date(); + const preparedSkills: PreparedDiscoveredSkill[] = []; + + for (const discovered of discoveredSkills) { + assertNotCancelled(); + assertGitHubSkillPackageManifest(discovered); + const skillMdPath = getSkillMdPath(discovered); + const skillMdBuffer = await fetchBlob({ + fetchFn, + token, + source, + sha: discovered.skillMd.sha, + }); + assertNotCancelled(); + assertGitHubBufferSize(skillMdBuffer, skillMdPath); + const prepared = await prepareRemoteSkill({ + deps, + source, + discovered, + skillMdContent: skillMdBuffer.toString('utf-8'), + commitSha: commit.sha, + syncedAt, + }); + preparedSkills.push({ discovered, prepared }); + } + + const discoveredUpstreamIds = new Set( + preparedSkills.map(({ discovered }) => makeUpstreamId(source, discovered.rootPath)), + ); + assertNoDuplicatePreparedSkillNames(source, preparedSkills); + const orderedPreparedSkills = orderPreparedSkillsForSafeStaleDeletes({ + source, + preparedSkills, + existingSyncedSkills: await getExistingSyncedSkills(), + discoveredUpstreamIds, + }); + + for (const { discovered, prepared } of orderedPreparedSkills) { + assertNotCancelled(); + const movedExisting = prepared.existing + ? null + : findMovedSourceSkill({ + source, + prepared, + existingSyncedSkills: await getExistingSyncedSkills(), + excludedUpstreamIds: discoveredUpstreamIds, + }); + const effectivePrepared: PreparedRemoteSkill = movedExisting + ? { ...prepared, existing: movedExisting } + : prepared; + seenUpstreamIds.add(makeUpstreamId(source, discovered.rootPath)); + if (effectivePrepared.existing) { + // Check for an external edit before mutating files, so a concurrently + // edited skill fails fast without leaving its bundled files partially + // rewritten to the upstream version. The post-file-sync check below + // still guards edits that land during the file sync itself. + const beforeFileSync = await deps.getSkillById(effectivePrepared.existing._id); + if (!beforeFileSync) { + throw new SkillSyncError( + 'SKILL_NOT_FOUND', + `Previously synced skill "${effectivePrepared.existing.name}" was removed`, + ); + } + if (hasExternalSkillEdit(effectivePrepared.existing, beforeFileSync)) { + throw new SkillSyncError( + 'SKILL_CONFLICT', + `Skill "${effectivePrepared.existing.name}" was modified during sync`, + ); + } + await ensurePublicViewer(deps, effectivePrepared.existing._id); + const previousFiles = await deps.listSkillFiles(effectivePrepared.existing._id); + const journal: SyncSkillFilesJournal = { staleFiles: [], savedFiles: [] }; + let fileCounts: SyncSkillFilesResult; + let staleConflictCleanup: + | Awaited> + | undefined; + try { + fileCounts = await syncSkillFiles({ + deps, + token, + source, + skill: effectivePrepared.existing, + discovered, + commitSha: commit.sha, + fetchFn, + assertNotCancelled, + journal, + }); + if (prepared.existing) { + staleConflictCleanup = await deleteNameConflictingStaleSkill({ + deps, + source, + prepared: effectivePrepared, + existingSyncedSkills: await getExistingSyncedSkills(), + discoveredUpstreamIds, + assertNotCancelled, + }); + existingSyncedSkills = staleConflictCleanup.remainingSkills; + counts.deletedSkillCount += staleConflictCleanup.deletedSkillCount; + counts.deletedFileCount += staleConflictCleanup.deletedFileCount; + } + await commitExistingRemoteSkillAfterFileSync( + deps, + { + ...effectivePrepared, + existing: effectivePrepared.existing, + }, + { forceCommit: fileCounts.syncedFileCount > 0 || fileCounts.deletedFileCount > 0 }, + ); + } catch (error) { + await restoreExistingSkillFiles({ + deps, + skill: effectivePrepared.existing, + previousFiles, + savedFiles: journal.savedFiles, + }).catch((cleanupError) => + logger.error( + '[GitHubSkillSync] Failed to restore existing skill files after sync failure:', + cleanupError, + ), + ); + if (staleConflictCleanup?.deletedSkill) { + await restoreDeletedSyncedSkill(deps, staleConflictCleanup.deletedSkill).catch( + (cleanupError) => + logger.error( + '[GitHubSkillSync] Failed to restore stale mirrored skill after sync failure:', + cleanupError, + ), + ); + } + throw error; + } + await cleanupStoredFiles({ + deps, + files: fileCounts.staleFiles, + logMessage: '[GitHubSkillSync] Failed to clean up replaced synced file:', + }); + if (staleConflictCleanup?.deletedSkill) { + await cleanupDeletedSyncedSkillFiles(deps, staleConflictCleanup.deletedSkill); + } + counts.syncedSkillCount++; + counts.syncedFileCount += fileCounts.syncedFileCount; + counts.deletedFileCount += fileCounts.deletedFileCount; + continue; + } + + const upserted = await commitRemoteSkill(deps, effectivePrepared); + const { skill } = upserted; + try { + const fileCounts = await syncSkillFiles({ + deps, + token, + source, + skill, + discovered, + commitSha: commit.sha, + fetchFn, + assertNotCancelled, + }); + await ensurePublicViewer(deps, skill._id); + counts.syncedSkillCount++; + counts.syncedFileCount += fileCounts.syncedFileCount; + counts.deletedFileCount += fileCounts.deletedFileCount; + } catch (error) { + await deleteSyncedSkill(deps, skill).catch((cleanupError) => + logger.error( + '[GitHubSkillSync] Failed to roll back partially synced skill:', + cleanupError, + ), + ); + throw error; + } + } + + const currentSyncedSkills = await deps.listSkillsBySource({ + source: PROVIDER, + sourceId: source.id, + }); + // Only mirror-delete skills owned by this source's tenant. With no + // configured tenantId under non-strict isolation, listSkillsBySource can + // return github skills across tenants, so without this guard an ambient sync + // could delete another tenant's mirrored skills. Absent tenantId is its own + // (ambient) bucket. + const sourceTenantId = source.tenantId ?? undefined; + for (const skill of currentSyncedSkills) { + assertNotCancelled(); + if ((skill.tenantId ?? undefined) !== sourceTenantId) { + continue; + } + const upstreamId = + skill.sourceMetadata && typeof skill.sourceMetadata.upstreamId === 'string' + ? skill.sourceMetadata.upstreamId + : ''; + if (seenUpstreamIds.has(upstreamId)) { + continue; + } + counts.deletedFileCount += await deleteSyncedSkill(deps, skill); + counts.deletedSkillCount++; + } + + return deps.upsertStatus( + makeStatusInput({ + source, + status: 'succeeded', + startedAt, + finishedAt: new Date(), + counts, + }), + ); + } catch (error) { + const sanitized = sanitizeError(error); + logger.error(`[GitHubSkillSync] Source "${source.id}" failed: ${sanitized.message}`); + return deps.upsertStatus( + makeStatusInput({ + source, + status: 'failed', + startedAt, + finishedAt: new Date(), + errorCode: sanitized.code, + errorMessage: sanitized.message, + }), + ); + } +} + +/** + * Runs a source sync inside its tenant's async context when `tenantId` is set, + * so the tenant-isolation mongoose hooks scope every skill/file/ACL read and + * write to that tenant (required under strict isolation). Storage writes also + * receive the tenant explicitly via `skill.tenantId`. Without a configured + * tenant the sync runs in the ambient context, preserving single-tenant behavior. + * + * The callback is `async` per the tenant-context contract so the ALS store + * propagates across every awaited Mongoose operation in `syncSource`. + */ +function syncSourceInTenantContext(params: { + deps: GitHubSkillSyncDeps; + source: SkillSyncGitHubSourceConfig; + fetchFn: FetchFn; + assertNotCancelled: AssertNotCancelled; +}): Promise { + if (!params.source.tenantId) { + return syncSource(params); + } + return tenantStorage.run({ tenantId: params.source.tenantId }, async () => syncSource(params)); +} + +function getGithubConfig(config: SkillSyncConfig | undefined): { + enabled: boolean; + intervalMinutes: number; + runOnStartup: boolean; + sources: SkillSyncGitHubSourceConfig[]; +} { + return { + enabled: config?.github?.enabled ?? false, + intervalMinutes: config?.github?.intervalMinutes ?? 60, + runOnStartup: config?.github?.runOnStartup ?? false, + sources: + config?.github?.sources.map((source) => ({ + ...source, + skillDiscoveryDepth: source.skillDiscoveryDepth ?? SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH, + })) ?? [], + }; +} + +export function createGitHubSkillSyncRunner(deps: GitHubSkillSyncDeps): GitHubSkillSyncRunner { + const fetchFn = deps.fetchFn ?? fetch; + const lockOwnerPrefix = deps.lockOwner ?? `${process.pid}`; + + async function getStatus(): Promise { + const github = getGithubConfig(await deps.getConfig()); + const allowServerCredentials = deps.allowServerCredentials !== false; + const [storedStatuses, credentials] = await Promise.all([ + deps.listStatuses(PROVIDER), + allowServerCredentials ? deps.listCredentials(PROVIDER) : Promise.resolve([]), + ]); + const statusBySourceId = new Map( + storedStatuses.map((status) => [makeStatusKey(status.sourceId, status.tenantId), status]), + ); + const credentialByKey = new Map( + credentials.map((credential) => [credential.credentialKey, credential]), + ); + const sources = github.sources.map((source) => { + const stored = statusBySourceId.get(makeStatusKey(source.id, source.tenantId)); + const credential = + allowServerCredentials && source.credentialKey + ? credentialByKey.get(source.credentialKey) + : null; + const tokenEnvVar = getTokenEnvVarName(source.token); + const envTokenPresent = + allowServerCredentials && tokenEnvVar ? Boolean(process.env[tokenEnvVar]?.trim()) : false; + return { + provider: PROVIDER, + sourceId: source.id, + tenantId: source.tenantId, + status: stored?.status ?? 'idle', + credentialKey: source.credentialKey, + credentialPresent: envTokenPresent || Boolean(credential), + owner: source.owner, + repo: source.repo, + ref: source.ref, + paths: source.paths, + startedAt: stored?.startedAt, + finishedAt: stored?.finishedAt, + lastSuccessAt: stored?.lastSuccessAt, + lastFailureAt: stored?.lastFailureAt, + errorCode: stored?.errorCode, + errorMessage: stored?.errorMessage, + syncedSkillCount: stored?.syncedSkillCount ?? 0, + syncedFileCount: stored?.syncedFileCount ?? 0, + deletedSkillCount: stored?.deletedSkillCount ?? 0, + deletedFileCount: stored?.deletedFileCount ?? 0, + createdAt: stored?.createdAt, + updatedAt: stored?.updatedAt, + } satisfies ISkillSyncStatus & { credentialPresent: boolean }; + }); + return { + enabled: github.enabled, + intervalMinutes: github.intervalMinutes, + runOnStartup: github.runOnStartup, + sources, + credentials, + fineGrainedTokenRecommendation: GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION, + }; + } + + async function runOnce(): Promise { + const github = getGithubConfig(await deps.getConfig()); + if (!github.enabled || github.sources.length === 0) { + return { status: 'skipped', message: 'GitHub skill sync is disabled', sources: [] }; + } + const allowServerCredentials = deps.allowServerCredentials !== false; + if (!allowServerCredentials) { + const status = await getStatus(); + if (!status.sources.some((source) => source.credentialPresent)) { + return { + status: 'skipped', + message: 'GitHub skill sync credentials are not available for this runner', + sources: status.sources, + }; + } + } + const lockOwner = `${lockOwnerPrefix}:${crypto.randomUUID().replace(/-/g, '').slice(0, 12)}`; + const acquired = await deps.tryAcquireLock({ + provider: PROVIDER, + lockOwner, + leaseMs: LOCK_LEASE_MS, + }); + if (!acquired) { + const status = await getStatus(); + return { + status: 'skipped', + message: 'GitHub skill sync is already running', + sources: status.sources, + }; + } + let lockLost = false; + const assertNotCancelled = () => { + if (lockLost) { + throw new SkillSyncError('SYNC_LOCK_LOST', 'GitHub skill sync lock was lost'); + } + }; + const refreshTimer = setInterval( + () => { + deps + .refreshLock({ + provider: PROVIDER, + lockOwner, + leaseMs: LOCK_LEASE_MS, + }) + .then((refreshed) => { + if (!refreshed) { + lockLost = true; + logger.warn('[GitHubSkillSync] Failed to refresh active sync lock'); + } + }) + .catch((error) => { + lockLost = true; + logger.error('[GitHubSkillSync] Failed to refresh active sync lock:', error); + }); + }, + Math.max(60_000, Math.floor(LOCK_LEASE_MS / 3)), + ); + refreshTimer.unref?.(); + try { + const sources: ISkillSyncStatus[] = []; + for (const source of github.sources) { + if (lockLost) { + break; + } + sources.push( + await syncSourceInTenantContext({ deps, source, fetchFn, assertNotCancelled }), + ); + } + const failed = sources.some((source) => source.status === 'failed'); + return { + status: failed || lockLost ? 'failed' : 'completed', + message: lockLost ? 'GitHub skill sync lock was lost' : undefined, + sources, + }; + } finally { + clearInterval(refreshTimer); + await deps.releaseLock({ provider: PROVIDER, lockOwner }); + } + } + + return { getStatus, runOnce }; +} diff --git a/packages/api/src/skills/sync/index.ts b/packages/api/src/skills/sync/index.ts new file mode 100644 index 0000000000..8cfe4a506b --- /dev/null +++ b/packages/api/src/skills/sync/index.ts @@ -0,0 +1,3 @@ +export * from './github'; +export * from './orchestrator'; +export * from './scheduler'; diff --git a/packages/api/src/skills/sync/orchestrator.spec.ts b/packages/api/src/skills/sync/orchestrator.spec.ts new file mode 100644 index 0000000000..231ba280d3 --- /dev/null +++ b/packages/api/src/skills/sync/orchestrator.spec.ts @@ -0,0 +1,339 @@ +import type { SkillSyncConfig } from 'librechat-data-provider'; +import type { SkillSyncTriggerRunnerFactoryInput } from './orchestrator'; +import type { GitHubSkillSyncRunner } from './github'; +import { createSkillSyncTriggerOrchestrator } from './orchestrator'; + +type RunnerStatus = Awaited>; +type RunnerRunResult = Awaited>; + +const source = { + id: 'tenant-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + tenantId: 'other-tenant', +}; + +function skillSync( + overrides: Partial['github']> = {}, +): SkillSyncConfig { + return { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [source], + ...overrides, + }, + }; +} + +function statusFromConfig( + config: SkillSyncConfig | undefined, + { allowServerCredentials = true }: { allowServerCredentials?: boolean } = {}, +): RunnerStatus { + const github = config?.github; + return { + enabled: github?.enabled ?? false, + intervalMinutes: github?.intervalMinutes ?? 60, + runOnStartup: github?.runOnStartup ?? false, + sources: + github?.sources.map((configuredSource) => ({ + provider: 'github', + sourceId: configuredSource.id, + tenantId: configuredSource.tenantId, + status: 'idle', + credentialKey: configuredSource.credentialKey, + credentialPresent: + allowServerCredentials && + Boolean(configuredSource.credentialKey || configuredSource.token), + owner: configuredSource.owner, + repo: configuredSource.repo, + ref: configuredSource.ref, + paths: configuredSource.paths, + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + errorCode: undefined, + errorMessage: undefined, + startedAt: undefined, + finishedAt: undefined, + lastSuccessAt: undefined, + lastFailureAt: undefined, + createdAt: undefined, + updatedAt: undefined, + })) ?? [], + credentials: [], + fineGrainedTokenRecommendation: 'Use a GitHub fine-grained personal access token.', + }; +} + +function withRunnableCredentials(status: RunnerStatus): RunnerStatus { + return { + ...status, + sources: status.sources.map((configuredSource) => ({ + ...configuredSource, + credentialPresent: true, + })), + }; +} + +function completedRun(): RunnerRunResult { + return { status: 'completed', sources: [] }; +} + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} + +async function flushPromises() { + await Promise.resolve(); + await Promise.resolve(); +} + +function createHarness( + options: { + status?: RunnerStatus; + runOnce?: () => Promise; + } = {}, +) { + const runners: Array<{ + input: SkillSyncTriggerRunnerFactoryInput; + runner: GitHubSkillSyncRunner; + }> = []; + const logger = { + warn: jest.fn(), + error: jest.fn(), + }; + const createRunner = jest.fn((input: SkillSyncTriggerRunnerFactoryInput) => { + const runner: GitHubSkillSyncRunner = { + getStatus: jest.fn( + async () => + options.status ?? + statusFromConfig(await input.getConfig(), { + allowServerCredentials: input.allowServerCredentials !== false, + }), + ), + runOnce: jest.fn(options.runOnce ?? (async () => completedRun())), + }; + runners.push({ input, runner }); + return runner; + }); + const orchestrator = createSkillSyncTriggerOrchestrator({ + createRunner, + logger, + }); + return { createRunner, logger, orchestrator, runners }; +} + +describe('createSkillSyncTriggerOrchestrator', () => { + it('starts request sync from resolved admin skillSync config and derives tenant from the request', async () => { + const config = skillSync(); + const { orchestrator, runners } = createHarness({ + status: withRunnableCredentials(statusFromConfig(config)), + }); + + const started = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + }); + + const requestConfig = await runners[0].input.getConfig(); + expect(started).toBe(true); + expect(runners[0].input.allowServerCredentials).toBe(false); + expect(runners[0].runner.runOnce).toHaveBeenCalledTimes(1); + expect(requestConfig?.github?.runOnStartup).toBe(false); + expect(requestConfig?.github?.sources[0]).toEqual( + expect.objectContaining({ id: 'tenant-skills', tenantId: 'tenant-a' }), + ); + }); + + it('does not auto-start request sync when only server credentials are configured', async () => { + const config = skillSync(); + const { orchestrator, runners } = createHarness(); + + const started = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + }); + + expect(started).toBe(false); + expect(runners[0].input.allowServerCredentials).toBe(false); + expect(runners[0].runner.runOnce).not.toHaveBeenCalled(); + }); + + it('starts request sync with server credentials when the caller opts in', async () => { + const config = skillSync(); + const { orchestrator, runners } = createHarness(); + + const started = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + skillSyncAllowServerCredentials: true, + }); + + expect(started).toBe(true); + expect(runners[0].input.allowServerCredentials).toBe(true); + expect(runners[0].runner.runOnce).toHaveBeenCalledTimes(1); + }); + + it('does not start request sync for base YAML skillSync config', async () => { + const config = skillSync(); + const { createRunner, orchestrator } = createHarness(); + + const started = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: { skillSync: config } }, + user: { tenantId: 'tenant-a' }, + }); + + expect(started).toBe(false); + expect(createRunner).not.toHaveBeenCalled(); + }); + + it('creates an admin request runner from resolved config without disabling startup runs', async () => { + const config = skillSync(); + const { orchestrator, runners } = createHarness(); + + const runner = orchestrator.getRunnerForAdminRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + skillSyncAllowServerCredentials: true, + }); + const runnerConfig = await runners[0].input.getConfig(); + + expect(runner).toBe(runners[0].runner); + expect(runners[0].input.allowServerCredentials).toBe(true); + expect(runnerConfig?.github?.runOnStartup).toBe(true); + expect(runnerConfig?.github?.sources[0]).toEqual( + expect.objectContaining({ id: 'tenant-skills', tenantId: 'tenant-a' }), + ); + }); + + it('preserves configured tenant scope for platform admin override runners', async () => { + const config = skillSync(); + const { orchestrator, runners } = createHarness(); + + orchestrator.getRunnerForAdminRequest({ + config: { skillSync: config, config: {} }, + user: {}, + skillSyncAllowServerCredentials: true, + }); + const runnerConfig = await runners[0].input.getConfig(); + + expect(runnerConfig?.github?.runOnStartup).toBe(true); + expect(runnerConfig?.github?.sources[0]).toEqual( + expect.objectContaining({ id: 'tenant-skills', tenantId: 'other-tenant' }), + ); + }); + + it('preserves configured tenant scope for admin base skillSync runs', async () => { + const config = skillSync(); + const { orchestrator, runners } = createHarness(); + + orchestrator.getRunnerForAdminRequest({ + config: { skillSync: config, config: { skillSync: config } }, + user: { tenantId: 'tenant-a' }, + skillSyncAllowServerCredentials: true, + }); + const runnerConfig = await runners[0].input.getConfig(); + + expect(runnerConfig?.github?.runOnStartup).toBe(true); + expect(runnerConfig?.github?.sources[0]).toEqual( + expect.objectContaining({ id: 'tenant-skills', tenantId: 'other-tenant' }), + ); + }); + + it('does not allow admin override runners to use server credentials by default', async () => { + const config = skillSync(); + const { orchestrator, runners } = createHarness(); + + orchestrator.getRunnerForAdminRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + }); + + expect(runners[0].input.allowServerCredentials).toBe(false); + }); + + it('does not start request sync when the configured source is already running', async () => { + const config = skillSync({ runOnStartup: false }); + const { orchestrator, runners } = createHarness({ + status: { + ...statusFromConfig(config), + sources: [ + { + ...statusFromConfig(config).sources[0], + status: 'running', + startedAt: new Date(), + }, + ], + }, + }); + + const started = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + }); + + expect(started).toBe(false); + expect(runners[0].runner.runOnce).not.toHaveBeenCalled(); + }); + + it('retries request sync when a running source status is stale', async () => { + const config = skillSync({ runOnStartup: false }); + const { orchestrator, runners } = createHarness({ + status: { + ...statusFromConfig(config), + sources: [ + { + ...statusFromConfig(config).sources[0], + status: 'running', + credentialPresent: true, + startedAt: new Date(Date.now() - 40 * 60 * 1000), + }, + ], + }, + }); + + const started = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + }); + + expect(started).toBe(true); + expect(runners[0].runner.runOnce).toHaveBeenCalledTimes(1); + }); + + it('suppresses duplicate request sync while an equivalent run is in flight', async () => { + const pendingRun = deferred(); + const config = skillSync({ runOnStartup: false }); + const { orchestrator, runners } = createHarness({ + status: withRunnableCredentials(statusFromConfig(config)), + runOnce: () => pendingRun.promise, + }); + + const first = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + }); + const second = await orchestrator.maybeRunForRequest({ + config: { skillSync: config, config: {} }, + user: { tenantId: 'tenant-a' }, + }); + + pendingRun.resolve(completedRun()); + await flushPromises(); + + expect(first).toBe(true); + expect(second).toBe(false); + expect(runners).toHaveLength(1); + expect(runners[0].runner.runOnce).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/api/src/skills/sync/orchestrator.ts b/packages/api/src/skills/sync/orchestrator.ts new file mode 100644 index 0000000000..05ecd64ded --- /dev/null +++ b/packages/api/src/skills/sync/orchestrator.ts @@ -0,0 +1,256 @@ +import { skillSyncConfigSchema } from 'librechat-data-provider'; +import type { SkillSyncConfig } from 'librechat-data-provider'; +import type { GitHubSkillSyncRunner } from './github'; + +const REQUEST_SYNC_MIN_INTERVAL_MS = 5 * 60 * 1000; +const REQUEST_SYNC_STALE_RUNNING_MS = 35 * 60 * 1000; + +type MaybePromise = T | Promise; + +export type SkillSyncAppConfigLike = { + skillSync?: unknown; + config?: { + skillSync?: unknown; + }; +}; + +export type SkillSyncRequestUser = { + tenantId?: string | null; +}; + +export type SkillSyncRequestLike = { + config?: SkillSyncAppConfigLike; + user?: SkillSyncRequestUser; + skillSyncAllowServerCredentials?: boolean; +}; + +type ResolvedSkillSyncConfig = NonNullable; +type ResolvedGitHubSkillSyncConfig = NonNullable; +type SkillSyncConfigWithGitHub = ResolvedSkillSyncConfig & { + github: ResolvedGitHubSkillSyncConfig; +}; + +type SkillSyncRunnerStatus = Awaited>; + +type SkillSyncTriggerLogger = { + warn: (message: string, metadata?: object) => void; + error: (message: string, error?: unknown) => void; +}; + +export type SkillSyncTriggerRunnerFactoryInput = { + getConfig: () => MaybePromise; + loadAppConfig: () => MaybePromise; + allowServerCredentials?: boolean; +}; + +export type SkillSyncTriggerOrchestratorDeps = { + createRunner: (input: SkillSyncTriggerRunnerFactoryInput) => GitHubSkillSyncRunner; + logger: SkillSyncTriggerLogger; + minIntervalMs?: number; + staleRunningMs?: number; + inFlight?: Set; +}; + +export type SkillSyncTriggerOrchestrator = { + getRunnerForAdminRequest: (request: SkillSyncRequestLike) => GitHubSkillSyncRunner; + maybeRunForRequest: (request: SkillSyncRequestLike) => Promise; +}; + +function parseSkillSyncConfig( + raw: unknown, + logger: SkillSyncTriggerLogger, +): SkillSyncConfig | undefined { + if (!raw || typeof raw !== 'object') { + return undefined; + } + const parsed = skillSyncConfigSchema.safeParse(raw); + if (!parsed.success) { + logger.warn('[GitHubSkillSync] Ignoring invalid skill sync config', { + issues: parsed.error.flatten(), + }); + return undefined; + } + return parsed.data; +} + +function hasGitHubConfig(config: SkillSyncConfig | undefined): config is SkillSyncConfigWithGitHub { + return Boolean(config?.github); +} + +function isSameSkillSyncConfig( + left: SkillSyncConfig | undefined, + right: SkillSyncConfig | undefined, +) { + return JSON.stringify(left ?? null) === JSON.stringify(right ?? null); +} + +function getRequestTenantId(user: SkillSyncRequestUser | undefined): string | undefined { + return typeof user?.tenantId === 'string' && user.tenantId ? user.tenantId : undefined; +} + +function withRequestTenant( + config: SkillSyncConfigWithGitHub, + user: SkillSyncRequestUser | undefined, + { disableRunOnStartup = false } = {}, +): SkillSyncConfig { + const tenantId = getRequestTenantId(user); + return { + ...config, + github: { + ...config.github, + ...(disableRunOnStartup ? { runOnStartup: false } : {}), + sources: config.github.sources.map((source) => ({ + ...source, + // Tenant-scoped requests derive their tenant from the request; platform + // requests have no tenant and preserve an explicitly configured source. + tenantId: tenantId ?? source.tenantId, + })), + }, + }; +} + +function getRequestSkillSyncConfig( + appConfig: SkillSyncAppConfigLike | undefined, + user: SkillSyncRequestUser | undefined, + logger: SkillSyncTriggerLogger, +): SkillSyncConfig | undefined { + const resolved = parseSkillSyncConfig(appConfig?.skillSync, logger); + if ( + !hasGitHubConfig(resolved) || + !resolved.github.enabled || + resolved.github.sources.length === 0 + ) { + return undefined; + } + + const base = parseSkillSyncConfig(appConfig?.config?.skillSync, logger); + if (isSameSkillSyncConfig(resolved, base)) { + return undefined; + } + + return withRequestTenant(resolved, user, { disableRunOnStartup: true }); +} + +function getAdminRequestSkillSyncConfig( + appConfig: SkillSyncAppConfigLike | undefined, + user: SkillSyncRequestUser | undefined, + logger: SkillSyncTriggerLogger, +): SkillSyncConfig | undefined { + const resolved = parseSkillSyncConfig(appConfig?.skillSync, logger); + if (!hasGitHubConfig(resolved)) { + return resolved; + } + + const base = parseSkillSyncConfig(appConfig?.config?.skillSync, logger); + if (isSameSkillSyncConfig(resolved, base)) { + return resolved; + } + + return withRequestTenant(resolved, user); +} + +function toTimestamp(value: unknown): number { + if (!value) { + return 0; + } + if (value instanceof Date) { + return value.getTime(); + } + const parsed = Date.parse(String(value)); + return Number.isNaN(parsed) ? 0 : parsed; +} + +function getLastAttemptAt(source: SkillSyncRunnerStatus['sources'][number]): number { + return Math.max( + toTimestamp(source.finishedAt), + toTimestamp(source.startedAt), + toTimestamp(source.updatedAt), + toTimestamp(source.lastSuccessAt), + toTimestamp(source.lastFailureAt), + ); +} + +function shouldRunRequestSync( + status: SkillSyncRunnerStatus, + { minIntervalMs, staleRunningMs }: { minIntervalMs: number; staleRunningMs: number }, +): boolean { + if (!status.enabled || status.sources.length === 0) { + return false; + } + const intervalMs = Math.max(minIntervalMs, (status.intervalMinutes ?? 60) * 60 * 1000); + const now = Date.now(); + return status.sources.some((source) => { + if (!source.credentialPresent) { + return false; + } + if (source.status === 'running') { + const startedAt = toTimestamp(source.startedAt); + return Boolean(startedAt && now - startedAt >= staleRunningMs); + } + const lastAttemptAt = getLastAttemptAt(source); + return !lastAttemptAt || now - lastAttemptAt >= intervalMs; + }); +} + +function getRequestSyncKey( + config: SkillSyncConfigWithGitHub, + user: SkillSyncRequestUser | undefined, +) { + const tenantId = user?.tenantId ?? ''; + const sources = config.github.sources + .map((source) => source.id) + .sort() + .join(','); + return `${tenantId}:${sources}`; +} + +export function createSkillSyncTriggerOrchestrator( + deps: SkillSyncTriggerOrchestratorDeps, +): SkillSyncTriggerOrchestrator { + const inFlight = deps.inFlight ?? new Set(); + const minIntervalMs = deps.minIntervalMs ?? REQUEST_SYNC_MIN_INTERVAL_MS; + const staleRunningMs = deps.staleRunningMs ?? REQUEST_SYNC_STALE_RUNNING_MS; + + function getRunnerForAdminRequest(request: SkillSyncRequestLike): GitHubSkillSyncRunner { + const config = getAdminRequestSkillSyncConfig(request.config, request.user, deps.logger); + return deps.createRunner({ + getConfig: async () => config, + loadAppConfig: async () => request.config, + allowServerCredentials: Boolean(request.skillSyncAllowServerCredentials), + }); + } + + async function maybeRunForRequest(request: SkillSyncRequestLike): Promise { + const config = getRequestSkillSyncConfig(request.config, request.user, deps.logger); + if (!hasGitHubConfig(config)) { + return false; + } + + const syncKey = getRequestSyncKey(config, request.user); + if (inFlight.has(syncKey)) { + return false; + } + + const requestRunner = deps.createRunner({ + getConfig: async () => config, + loadAppConfig: async () => request.config, + allowServerCredentials: Boolean(request.skillSyncAllowServerCredentials), + }); + const status = await requestRunner.getStatus(); + if (!shouldRunRequestSync(status, { minIntervalMs, staleRunningMs })) { + return false; + } + + inFlight.add(syncKey); + void requestRunner + .runOnce() + .catch((error) => deps.logger.error('[GitHubSkillSync] Request-scoped sync failed:', error)) + .finally(() => inFlight.delete(syncKey)); + return true; + } + + return { + getRunnerForAdminRequest, + maybeRunForRequest, + }; +} diff --git a/packages/api/src/skills/sync/scheduler.spec.ts b/packages/api/src/skills/sync/scheduler.spec.ts new file mode 100644 index 0000000000..da9eb0fff4 --- /dev/null +++ b/packages/api/src/skills/sync/scheduler.spec.ts @@ -0,0 +1,86 @@ +import type { SkillSyncConfig } from 'librechat-data-provider'; +import type { GitHubSkillSyncRunner } from './github'; +import { SKILL_SYNC_MAX_TIMER_INTERVAL_MINUTES, startGitHubSkillSyncScheduler } from './scheduler'; +import { __resetShutdownStateForTests } from '~/app/shutdown'; + +const source = { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + credentialKey: 'github-skills-prod', +}; + +function config(intervalMinutes: number, enabled = true): SkillSyncConfig { + return { + github: { + enabled, + intervalMinutes, + runOnStartup: false, + sources: [source], + }, + }; +} + +function runner(): GitHubSkillSyncRunner { + return { + getStatus: jest.fn(), + runOnce: jest.fn(async () => ({ status: 'completed', sources: [] })), + } as unknown as GitHubSkillSyncRunner; +} + +async function flushPromises() { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +describe('startGitHubSkillSyncScheduler', () => { + beforeEach(() => { + jest.useFakeTimers(); + __resetShutdownStateForTests(); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + __resetShutdownStateForTests(); + }); + + it('clamps oversized intervals before scheduling a timer', async () => { + const setTimeoutSpy = jest.spyOn(global, 'setTimeout'); + const scheduler = startGitHubSkillSyncScheduler({ + getConfig: () => config(Number.MAX_SAFE_INTEGER), + runner: runner(), + }); + + await flushPromises(); + + expect(setTimeoutSpy).toHaveBeenLastCalledWith( + expect.any(Function), + SKILL_SYNC_MAX_TIMER_INTERVAL_MINUTES * 60 * 1000, + ); + scheduler.stop(); + }); + + it('uses fresh config when scheduling the next run', async () => { + let intervalMinutes = 5; + const skillRunner = runner(); + const setTimeoutSpy = jest.spyOn(global, 'setTimeout'); + const scheduler = startGitHubSkillSyncScheduler({ + getConfig: () => config(intervalMinutes), + runner: skillRunner, + }); + await flushPromises(); + + intervalMinutes = 10; + await jest.advanceTimersByTimeAsync(5 * 60 * 1000); + await flushPromises(); + + expect(skillRunner.runOnce).toHaveBeenCalledTimes(1); + expect(setTimeoutSpy).toHaveBeenLastCalledWith(expect.any(Function), 10 * 60 * 1000); + scheduler.stop(); + }); +}); diff --git a/packages/api/src/skills/sync/scheduler.ts b/packages/api/src/skills/sync/scheduler.ts new file mode 100644 index 0000000000..7aad2d7bab --- /dev/null +++ b/packages/api/src/skills/sync/scheduler.ts @@ -0,0 +1,103 @@ +import { logger } from '@librechat/data-schemas'; +import type { SkillSyncConfig } from 'librechat-data-provider'; +import type { GitHubSkillSyncRunner } from './github'; +import { registerShutdownTask } from '~/app/shutdown'; + +const NODE_TIMER_MAX_MS = 2147483647; +const SKILL_SYNC_MIN_INTERVAL_MINUTES = 5; +export const SKILL_SYNC_MAX_TIMER_INTERVAL_MINUTES: number = Math.floor(NODE_TIMER_MAX_MS / 60_000); + +type MaybePromise = T | Promise; + +export type GitHubSkillSyncScheduler = { + stop: () => void; +}; + +function normalizeIntervalMinutes(value: unknown): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return 60; + } + return Math.min( + SKILL_SYNC_MAX_TIMER_INTERVAL_MINUTES, + Math.max(SKILL_SYNC_MIN_INTERVAL_MINUTES, Math.floor(value)), + ); +} + +function getSources(config: SkillSyncConfig | undefined) { + const sources = config?.github?.sources; + return Array.isArray(sources) ? sources : []; +} + +export function startGitHubSkillSyncScheduler(params: { + getConfig: () => MaybePromise; + runner: GitHubSkillSyncRunner; +}): GitHubSkillSyncScheduler { + let stopped = false; + let timer: NodeJS.Timeout | undefined; + + const getConfig = async (): Promise => { + try { + return await params.getConfig(); + } catch (error) { + logger.error('[GitHubSkillSync] Failed to load scheduler config:', error); + return undefined; + } + }; + + const scheduleNext = (config: SkillSyncConfig | undefined) => { + if (stopped) { + return; + } + const delayMs = normalizeIntervalMinutes(config?.github?.intervalMinutes) * 60 * 1000; + timer = setTimeout(tick, delayMs); + timer.unref?.(); + }; + + const runIfEnabled = async () => { + const config = await getConfig(); + const github = config?.github; + if (!github?.enabled || getSources(config).length === 0) { + return config; + } + try { + await params.runner.runOnce(); + } catch (error) { + logger.error('[GitHubSkillSync] Scheduled run failed:', error); + } + return getConfig(); + }; + + async function tick() { + if (stopped) { + return; + } + const config = await runIfEnabled(); + scheduleNext(config); + } + + const scheduler = { + stop: () => { + stopped = true; + if (timer) { + clearTimeout(timer); + timer = undefined; + } + }, + }; + + void (async () => { + const config = await getConfig(); + if (config?.github?.enabled && config.github.runOnStartup && getSources(config).length > 0) { + void params.runner.runOnce().catch((error) => { + logger.error('[GitHubSkillSync] Scheduled startup run failed:', error); + }); + } + scheduleNext(config); + })(); + + registerShutdownTask('github skill sync scheduler', () => { + scheduler.stop(); + }); + + return scheduler; +} diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index 7186d87cf8..3364a684ba 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.8.503", + "version": "0.8.504", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.mjs", diff --git a/packages/data-provider/specs/config-schemas.spec.ts b/packages/data-provider/specs/config-schemas.spec.ts index a285aebe0b..e7a3f3dc53 100644 --- a/packages/data-provider/specs/config-schemas.spec.ts +++ b/packages/data-provider/specs/config-schemas.spec.ts @@ -8,6 +8,7 @@ import { interfaceSchema, fileStorageSchema, fileStrategiesSchema, + SKILL_SYNC_MAX_INTERVAL_MINUTES, summarizationTriggerSchema, summarizationConfigSchema, MAX_SUBAGENTS, @@ -681,6 +682,294 @@ describe('configSchema fileStrategy', () => { }); }); +describe('configSchema skillSync', () => { + it('accepts a GitHub skill sync source with explicit paths and credential key', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills', '.'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }, + }); + expect(result.success).toBe(true); + expect(result.data?.skillSync?.github?.sources[0]?.paths).toEqual(['skills', '']); + expect(result.data?.skillSync?.github?.sources[0]?.skillDiscoveryDepth).toBeUndefined(); + }); + + it('accepts a GitHub skill sync source with an env-backed token and discovery depth', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'mattpocock-skills', + owner: 'mattpocock', + repo: 'skills', + paths: ['skills'], + skillDiscoveryDepth: 3, + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }, + }); + expect(result.success).toBe(true); + expect(result.data?.skillSync?.github?.sources[0]?.token).toBe('${GITHUB_SKILLS_TOKEN}'); + expect(result.data?.skillSync?.github?.sources[0]?.skillDiscoveryDepth).toBe(3); + }); + + it('accepts an optional tenantId on a GitHub skill sync source', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + credentialKey: 'github-skills-prod', + tenantId: 'tenant-a', + }, + ], + }, + }, + }); + expect(result.success).toBe(true); + expect(result.data?.skillSync?.github?.sources[0]?.tenantId).toBe('tenant-a'); + }); + + it('rejects the reserved system tenant id on a GitHub skill sync source', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + credentialKey: 'github-skills-prod', + tenantId: '__SYSTEM__', + }, + ], + }, + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects enabled GitHub skill sync without sources', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [], + }, + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects GitHub skill sync intervals below five minutes', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + intervalMinutes: 4, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects GitHub skill sync intervals above the Node timer limit', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + intervalMinutes: SKILL_SYNC_MAX_INTERVAL_MINUTES + 1, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects unsafe GitHub skill sync paths and credential keys', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['../skills'], + credentialKey: '../token', + }, + ], + }, + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects GitHub skill sync sources without credentials or with literal tokens', () => { + const missingCredential = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + }, + ], + }, + }, + }); + const literalToken = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + token: 'github_pat_secret', + }, + ], + }, + }, + }); + const duplicateCredentialSources = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + credentialKey: 'github-skills-prod', + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }, + }); + expect(missingCredential.success).toBe(false); + expect(literalToken.success).toBe(false); + expect(duplicateCredentialSources.success).toBe(false); + }); + + it('rejects GitHub skill sync discovery depths outside the allowed range', () => { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + paths: ['skills'], + credentialKey: 'github-skills-prod', + skillDiscoveryDepth: 11, + }, + ], + }, + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects malformed GitHub skill sync refs during config validation', () => { + const invalidRefs = [ + 'feature branch', + 'release:2026', + 'bad?ref', + 'bad*ref', + '[bad]', + '@{upstream}', + 'main.lock', + ]; + + for (const ref of invalidRefs) { + const result = configSchema.safeParse({ + version: '1.3.11', + skillSync: { + github: { + enabled: true, + sources: [ + { + id: 'librechat-skills', + owner: 'LibreChat', + repo: 'skills', + ref, + paths: ['skills'], + credentialKey: 'github-skills-prod', + }, + ], + }, + }, + }); + expect(result.success).toBe(false); + } + }); +}); + describe('interfaceSchema', () => { it('silently strips removed legacy fields', () => { const result = interfaceSchema.parse({ diff --git a/packages/data-provider/src/api-endpoints.ts b/packages/data-provider/src/api-endpoints.ts index 6cec5b0b51..bc3107d7ca 100644 --- a/packages/data-provider/src/api-endpoints.ts +++ b/packages/data-provider/src/api-endpoints.ts @@ -1,7 +1,7 @@ import type { StartupConfigContext } from './config'; import type { AssistantsEndpoint } from './schemas'; -import * as q from './types/queries'; import { ResourceType } from './accessPermissions'; +import * as q from './types/queries'; let BASE_URL = ''; if ( @@ -404,6 +404,12 @@ export const skillFiles = (id: string) => `${getSkill(id)}/files`; export const skillFile = (id: string, relativePath: string) => `${skillFiles(id)}/${encodeURIComponent(relativePath)}`; +export const adminSkillsSync = () => `${BASE_URL}/api/admin/skills/sync`; +export const adminSkillsSyncStatus = () => `${adminSkillsSync()}/status`; +export const adminSkillsSyncRun = () => `${adminSkillsSync()}/run`; +export const adminSkillsSyncCredential = (credentialKey: string) => + `${adminSkillsSync()}/credentials/${encodeURIComponent(credentialKey)}`; + /** * Skill filesystem tree (phase 2). URL shape mirrors the original UI PR so * the tree hooks keep their call surface. `path` is pre-encoded by the diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index a431031050..6309cfc279 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -19,6 +19,8 @@ export { MAX_SUBAGENTS } from './limits'; export const defaultSocialLogins = ['google', 'facebook', 'openid', 'github', 'discord', 'saml']; +export const BASE_ONLY_CONFIG_SECTIONS = [] as const; + export const defaultRetrievalModels = [ 'gpt-4o', 'o1-preview-2024-09-12', @@ -244,6 +246,192 @@ export const cloudfrontConfigSchema = z export type CloudFrontConfig = z.infer; +const skillSyncIdentifierSchema = z + .string() + .min(1) + .max(64) + .regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, { + message: + 'must start with a letter or digit and contain only letters, digits, underscores, or hyphens', + }); + +export const SKILL_SYNC_MIN_INTERVAL_MINUTES = 5; +export const SKILL_SYNC_MAX_INTERVAL_MINUTES = Math.floor(2147483647 / 60_000); +export const SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH = 2; +export const SKILL_SYNC_MAX_DISCOVERY_DEPTH = 10; + +const skillSyncGitHubOwnerSchema = z + .string() + .min(1) + .max(39) + .regex(/^[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?$/, { + message: 'must be a valid GitHub owner name', + }); + +const skillSyncGitHubRepoSchema = z + .string() + .min(1) + .max(100) + .regex(/^[a-zA-Z0-9._-]+$/, { + message: 'must be a valid GitHub repository name', + }); + +const invalidGitRefChars = new Set(['~', '^', ':', '?', '*', '[']); + +function hasInvalidGitRefCharacter(value: string): boolean { + for (const char of value) { + const code = char.charCodeAt(0); + if (code <= 32 || code === 127 || invalidGitRefChars.has(char)) { + return true; + } + } + return false; +} + +const skillSyncGitHubRefSchema = z + .string() + .min(1) + .max(255) + .refine((value) => !value.startsWith('/') && !value.endsWith('/'), { + message: 'must not start or end with a slash', + }) + .refine((value) => !value.includes('..') && !value.includes('//') && !value.includes('\\'), { + message: 'must not contain traversal segments, empty path segments, or backslashes', + }) + .refine((value) => !value.includes('@{') && value !== '@', { + message: 'must not contain invalid Git ref syntax', + }) + .refine((value) => !value.endsWith('.'), { + message: 'must not end with a dot', + }) + .refine((value) => !hasInvalidGitRefCharacter(value), { + message: 'must not contain invalid Git ref characters', + }) + .refine( + (value) => + value + .split('/') + .every((segment) => segment && !segment.startsWith('.') && !segment.endsWith('.lock')), + { + message: 'must contain valid Git ref path segments', + }, + ); + +const skillSyncPathSchema = z + .string() + .max(500) + .refine((value) => value.trim().length > 0, { message: 'must not be empty' }) + .transform((value) => { + const trimmed = value.trim().replace(/^\/+|\/+$/g, ''); + return trimmed === '.' ? '' : trimmed; + }) + .refine((value) => !value.includes('\\') && !value.includes('..'), { + message: 'must not contain traversal segments or backslashes', + }) + .refine((value) => value === '' || /^[a-zA-Z0-9._\-/]+$/.test(value), { + message: 'must contain only letters, digits, dots, underscores, hyphens, and slashes', + }) + .refine( + (value) => + value === '' || value.split('/').every((segment) => segment.length > 0 && segment !== '.'), + { + message: 'must not contain empty or dot path segments', + }, + ); + +const skillSyncTokenReferenceSchema = z + .string() + .trim() + .regex(/^\$\{[A-Za-z_][A-Za-z0-9_]*\}$/, { + message: 'must be an environment variable reference like ${GITHUB_SKILLS_TOKEN}', + }); + +/** + * Tenant that owns the skills mirrored from a source. When set, the sync runner + * executes that source's database writes inside the tenant's async context so + * synced skills are created, listed, and shared within the tenant under strict + * tenant isolation. Mirrors the request tenant-id contract: no reserved system id. + */ +const skillSyncTenantIdSchema = z + .string() + .max(128) + .refine((value) => /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(value), { + message: 'must be a valid tenant id', + }) + .refine((value) => value !== '__SYSTEM__', { + message: 'must not be the reserved system tenant id', + }); + +export const skillSyncGitHubSourceSchema = z + .object({ + id: skillSyncIdentifierSchema, + owner: skillSyncGitHubOwnerSchema, + repo: skillSyncGitHubRepoSchema, + ref: skillSyncGitHubRefSchema.default('main'), + paths: z.array(skillSyncPathSchema).min(1), + skillDiscoveryDepth: z.number().int().min(0).max(SKILL_SYNC_MAX_DISCOVERY_DEPTH).optional(), + credentialKey: skillSyncIdentifierSchema.optional(), + token: skillSyncTokenReferenceSchema.optional(), + tenantId: skillSyncTenantIdSchema.optional(), + }) + .superRefine((source, ctx) => { + if (!source.credentialKey && !source.token) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['credentialKey'], + message: 'Either credentialKey or token is required', + }); + } + if (source.credentialKey && source.token) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['token'], + message: 'Use either credentialKey or token, not both', + }); + } + }); + +export const skillSyncConfigSchema = z + .object({ + github: z + .object({ + enabled: z.boolean().default(false), + intervalMinutes: z + .number() + .int() + .min(SKILL_SYNC_MIN_INTERVAL_MINUTES) + .max(SKILL_SYNC_MAX_INTERVAL_MINUTES) + .default(60), + runOnStartup: z.boolean().default(false), + sources: z.array(skillSyncGitHubSourceSchema).default([]), + }) + .superRefine((github, ctx) => { + if (github.enabled && github.sources.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['sources'], + message: 'At least one GitHub source is required when skill sync is enabled', + }); + } + const seen = new Set(); + for (const source of github.sources) { + if (seen.has(source.id)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['sources'], + message: `Duplicate GitHub skill sync source id "${source.id}"`, + }); + } + seen.add(source.id); + } + }) + .optional(), + }) + .optional(); + +export type SkillSyncConfig = z.infer; +export type SkillSyncGitHubSourceConfig = z.infer; + // Helper type to extract the shape of the Zod object schema type SchemaShape = T extends z.ZodObject ? U : never; @@ -1462,6 +1650,7 @@ export const configSchema = z.object({ webSearch: webSearchSchema.optional(), memory: memorySchema.optional(), summarization: summarizationConfigSchema.optional(), + skillSync: skillSyncConfigSchema, secureImageLinks: z.boolean().optional(), imageOutputType: z.nativeEnum(EImageOutputType).default(EImageOutputType.PNG), includedTools: z.array(z.string()).optional(), diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.ts index f1d66e42d9..95d362ac34 100644 --- a/packages/data-provider/src/data-service.ts +++ b/packages/data-provider/src/data-service.ts @@ -1080,6 +1080,29 @@ export const updateSkillNodeContent = (variables: { }); }; +export function getGitHubSkillSyncStatus(): Promise { + return request.get(endpoints.adminSkillsSyncStatus()); +} + +export function runGitHubSkillSync(): Promise { + return request.post(endpoints.adminSkillsSyncRun()); +} + +export function setGitHubSkillSyncCredential(variables: { + credentialKey: string; + token: string; +}): Promise { + return request.put(endpoints.adminSkillsSyncCredential(variables.credentialKey), { + token: variables.token, + } satisfies sk.TGitHubSkillSyncCredentialUpdateRequest); +} + +export function deleteGitHubSkillSyncCredential( + credentialKey: string, +): Promise<{ credentialKey: string; deleted: boolean }> { + return request.delete(endpoints.adminSkillsSyncCredential(credentialKey)); +} + /* Roles */ export function listRoles(): Promise { return request.get(`${endpoints.adminRoles()}?limit=200`); diff --git a/packages/data-provider/src/types/skills.ts b/packages/data-provider/src/types/skills.ts index 302c865213..487edc29dc 100644 --- a/packages/data-provider/src/types/skills.ts +++ b/packages/data-provider/src/types/skills.ts @@ -27,7 +27,7 @@ export const SKILL_NAME_PATTERN = /^[a-z0-9][a-z0-9-]*$/; * `inline` means the skill was authored directly in LibreChat. * `deployment` means the skill was loaded from the server's configured * deployment skill directory and is not persisted as a Skill document. - * `github` / `notion` are reserved for future sync integrations. + * `github` is populated by admin-configured GitHub skill sync; `notion` is reserved. */ export type SkillSource = 'inline' | 'deployment' | 'github' | 'notion'; @@ -60,11 +60,25 @@ export type SkillFrontmatter = { * Provenance metadata for skills that originated from an external source * (e.g. a GitHub commit SHA or a Notion page id). * - * Reserved for phase 2+ external sync — no code path currently populates this - * in phase 1, but the column exists so a future sync worker can use it - * without a schema migration. + * Populated by external sync workers with upstream identifiers such as source + * ids, paths, and commit/blob SHAs. */ -export type SkillSourceMetadata = Record; +export type SkillSourceMetadata = + | Record + | { + provider: 'github'; + sourceId: string; + upstreamId: string; + owner: string; + repo: string; + ref: string; + skillPath: string; + commitSha?: string; + skillBlobSha?: string; + syncedAt?: string; + syncStatus?: 'synced' | 'failed'; + error?: string; + }; /** * A non-blocking coaching hint surfaced alongside a successful create/update @@ -95,7 +109,7 @@ export type TSkillWarning = { * (those live as top-level columns). Validated strictly against a known * key set server-side. * - `source`/`sourceMetadata` identify whether the row is user-authored, - * deployment-provided, or reserved for a future sync provider. + * deployment-provided, or mirrored from an external source such as GitHub. */ export type TSkill = { _id: string; @@ -183,6 +197,7 @@ export type TSkillFile = { isExecutable: boolean; author: string; tenantId?: string; + sourceMetadata?: Record; /** Lazily cached text content (≤ 512 KB). Excluded from list responses. */ content?: string; /** Set on first read. `true` prevents repeated storage reads for non-text files. */ @@ -191,6 +206,59 @@ export type TSkillFile = { updatedAt: string; }; +export type TGitHubSkillSyncCredentialSummary = { + provider: 'github'; + credentialKey: string; + credentialPresent: boolean; + tokenFingerprint?: string; + updatedAt?: string; + createdAt?: string; +}; + +export type TGitHubSkillSyncSourceStatus = { + provider: 'github'; + sourceId: string; + tenantId?: string; + status: 'idle' | 'running' | 'succeeded' | 'failed' | 'skipped'; + credentialKey?: string; + credentialPresent: boolean; + owner?: string; + repo?: string; + ref?: string; + paths?: string[]; + startedAt?: string; + finishedAt?: string; + lastSuccessAt?: string; + lastFailureAt?: string; + errorCode?: string; + errorMessage?: string; + syncedSkillCount: number; + syncedFileCount: number; + deletedSkillCount: number; + deletedFileCount: number; + updatedAt?: string; + createdAt?: string; +}; + +export type TGitHubSkillSyncStatusResponse = { + enabled: boolean; + intervalMinutes: number; + runOnStartup: boolean; + sources: TGitHubSkillSyncSourceStatus[]; + credentials: TGitHubSkillSyncCredentialSummary[]; + fineGrainedTokenRecommendation: string; +}; + +export type TGitHubSkillSyncCredentialUpdateRequest = { + token: string; +}; + +export type TGitHubSkillSyncManualRunResponse = { + status: 'started' | 'skipped' | 'completed' | 'failed'; + message?: string; + sources?: TGitHubSkillSyncSourceStatus[]; +}; + /** Request body for POST `/api/skills`. */ export type TCreateSkill = { name: string; diff --git a/packages/data-schemas/src/app/resolution.spec.ts b/packages/data-schemas/src/app/resolution.spec.ts index 80189c2f96..d3020c0707 100644 --- a/packages/data-schemas/src/app/resolution.spec.ts +++ b/packages/data-schemas/src/app/resolution.spec.ts @@ -1,6 +1,6 @@ import { INTERFACE_PERMISSION_FIELDS, PermissionTypes } from 'librechat-data-provider'; -import { mergeConfigOverrides } from './resolution'; import type { AppConfig, IConfig } from '~/types'; +import { mergeConfigOverrides } from './resolution'; function fakeConfig( overrides: Record, @@ -347,6 +347,66 @@ describe('mergeConfigOverrides', () => { expect(iface.parameters).toBe(true); }); + it('merges skillSync config sections from DB overrides', () => { + const base = { + skillSync: { + github: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'base-source', + owner: 'LibreChat', + repo: 'skills', + ref: 'main', + paths: ['skills'], + token: '${GITHUB_SKILLS_TOKEN}', + }, + ], + }, + }, + interfaceConfig: { modelSelect: true }, + } as unknown as AppConfig; + + const configs = [ + fakeConfig( + { + skillSync: { + github: { + enabled: false, + sources: [ + { + id: 'override-source', + owner: 'other', + repo: 'skills', + paths: ['skills'], + token: '${OTHER_TOKEN}', + }, + ], + }, + }, + interface: { modelSelect: false }, + }, + 10, + ), + ]; + + const result = mergeConfigOverrides(base, configs); + + expect(result.skillSync?.github?.enabled).toBe(false); + expect(result.skillSync?.github?.sources).toEqual([ + { + id: 'override-source', + owner: 'other', + repo: 'skills', + paths: ['skills'], + token: '${OTHER_TOKEN}', + }, + ]); + expect(result.interfaceConfig?.modelSelect).toBe(false); + }); + it('preserves UI sub-keys in composite permission fields like mcpServers', () => { const base = { interfaceConfig: {}, diff --git a/packages/data-schemas/src/app/resolution.ts b/packages/data-schemas/src/app/resolution.ts index 5acc37c5cc..4f17acac2c 100644 --- a/packages/data-schemas/src/app/resolution.ts +++ b/packages/data-schemas/src/app/resolution.ts @@ -1,4 +1,8 @@ -import { INTERFACE_PERMISSION_FIELDS, PERMISSION_SUB_KEYS } from 'librechat-data-provider'; +import { + BASE_ONLY_CONFIG_SECTIONS, + INTERFACE_PERMISSION_FIELDS, + PERMISSION_SUB_KEYS, +} from 'librechat-data-provider'; import type { TCustomConfig } from 'librechat-data-provider'; import type { AppConfig, IConfig } from '~/types'; @@ -6,6 +10,7 @@ type AnyObject = { [key: string]: unknown }; const MAX_MERGE_DEPTH = 10; const UNSAFE_KEYS = new Set(['__proto__', 'constructor', 'prototype']); +const BASE_ONLY_OVERRIDE_SECTIONS = new Set(BASE_ONLY_CONFIG_SECTIONS); /** * Paths within the config tree where arrays of objects should be merged by @@ -199,6 +204,9 @@ export function mergeConfigOverrides(baseConfig: AppConfig, configs: IConfig[]): if (config.overrides && typeof config.overrides === 'object') { const remapped: AnyObject = {}; for (const [key, value] of Object.entries(config.overrides)) { + if (BASE_ONLY_OVERRIDE_SECTIONS.has(key)) { + continue; + } const mappedKey = OVERRIDE_KEY_MAP[key as keyof typeof OVERRIDE_KEY_MAP] ?? key; if ( key === 'interface' && diff --git a/packages/data-schemas/src/app/service.ts b/packages/data-schemas/src/app/service.ts index 4d41a824f4..fe3ae1b007 100644 --- a/packages/data-schemas/src/app/service.ts +++ b/packages/data-schemas/src/app/service.ts @@ -1,6 +1,7 @@ import { EModelEndpoint, getConfigDefaults, + skillSyncConfigSchema, summarizationConfigSchema, } from 'librechat-data-provider'; import type { TCustomConfig, FileSources, DeepPartial } from 'librechat-data-provider'; @@ -51,6 +52,21 @@ export function loadSummarizationConfig( }; } +export function loadSkillSyncConfig(config: DeepPartial): AppConfig['skillSync'] { + const raw = config.skillSync; + if (!raw || typeof raw !== 'object') { + return undefined; + } + + const parsed = skillSyncConfigSchema.safeParse(raw); + if (!parsed.success) { + logger.warn('[AppService] Invalid skill sync config', parsed.error.flatten()); + return undefined; + } + + return parsed.data; +} + export type Paths = { root: string; uploads: string; @@ -83,6 +99,7 @@ export const AppService = async (params?: { const webSearch = loadWebSearchConfig(config.webSearch); const memory = loadMemoryConfig(config.memory); const summarization = loadSummarizationConfig(config); + const skillSync = loadSkillSyncConfig(config); const filteredTools = config.filteredTools; const includedTools = config.includedTools; const fileStrategy = (config.fileStrategy ?? configDefaults.fileStrategy) as @@ -120,6 +137,7 @@ export const AppService = async (params?: { speech, actions, balance, + skillSync, webSearch, mcpSettings, fileStrategy, diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index abedc18e12..6bab02a4f4 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -1,10 +1,10 @@ +import type { RoleMethods, RoleDeps } from './role'; import { createSessionMethods, DEFAULT_REFRESH_TOKEN_EXPIRY, type SessionMethods } from './session'; +import { createUserMethods, DEFAULT_SESSION_EXPIRY, type UserMethods } from './user'; import { createTokenMethods, type TokenMethods } from './token'; import { createRoleMethods, RoleConflictError } from './role'; -import type { RoleMethods, RoleDeps } from './role'; -import { createUserMethods, DEFAULT_SESSION_EXPIRY, type UserMethods } from './user'; -import { createKeyMethods, type KeyMethods } from './key'; import { createFileMethods, type FileMethods } from './file'; +import { createKeyMethods, type KeyMethods } from './key'; /* Memories */ import { createMemoryMethods, type MemoryMethods } from './memory'; /* Agent Categories */ @@ -77,6 +77,12 @@ import { type UpdateSkillResult, type ValidationIssue, } from './skill'; +import { createSkillSyncMethods, type SkillSyncMethods } from './skillSync'; +import type { + SkillSyncStatusInput, + SkillSyncCredentialSummary, + UpsertSkillSyncCredentialInput, +} from './skillSync'; /* Tier 5 — Agent */ import { createAgentMethods, type AgentMethods, type AgentDeps } from './agent'; /* Config */ @@ -127,6 +133,7 @@ export type AllMethods = UserMethods & SpendTokensMethods & PromptMethods & SkillMethods & + SkillSyncMethods & AgentMethods & ConfigMethods; @@ -255,6 +262,7 @@ export function createMethods( ...spendTokensMethods, ...promptMethods, ...skillMethods, + ...createSkillSyncMethods(mongoose), /* Tier 5 */ ...agentMethods, /* Config */ @@ -303,6 +311,10 @@ export type { ListSkillsByAccessResult, UpdateSkillResult, ValidationIssue, + SkillSyncStatusInput, + SkillSyncCredentialSummary, + UpsertSkillSyncCredentialInput, + SkillSyncMethods, AgentMethods, ConfigMethods, }; diff --git a/packages/data-schemas/src/methods/skill.spec.ts b/packages/data-schemas/src/methods/skill.spec.ts index 0db2e69ce5..60451d821b 100644 --- a/packages/data-schemas/src/methods/skill.spec.ts +++ b/packages/data-schemas/src/methods/skill.spec.ts @@ -1,4 +1,5 @@ import mongoose from 'mongoose'; +import { logger, createModels } from '..'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { SystemRoles, @@ -7,7 +8,6 @@ import { PrincipalType, PermissionBits, } from 'librechat-data-provider'; -import { createAclEntryMethods } from './aclEntry'; import { validateSkillName, validateSkillDescription, @@ -17,7 +17,7 @@ import { inferSkillFileCategory, deriveStructuredFrontmatterFields, } from './skill'; -import { logger, createModels } from '..'; +import { createAclEntryMethods } from './aclEntry'; import { createMethods } from './index'; logger.silent = true; @@ -139,6 +139,23 @@ function makeSkillInput(overrides: Record = {}) { }; } +describe('Skill schema indexes', () => { + it('supports GitHub sync source metadata lookups', () => { + const indexSpecs = Skill.schema.indexes().map(([fields]) => fields); + + expect(indexSpecs).toContainEqual({ + source: 1, + 'sourceMetadata.upstreamId': 1, + tenantId: 1, + }); + expect(indexSpecs).toContainEqual({ + source: 1, + 'sourceMetadata.sourceId': 1, + tenantId: 1, + }); + }); +}); + describe('skill validation helpers', () => { it('rejects names starting with reserved brand prefixes', () => { expect(validateSkillName('anthropic-helper').some((i) => i.code === 'RESERVED_PREFIX')).toBe( @@ -516,6 +533,48 @@ describe('Skill CRUD methods', () => { expect(await SkillFile.countDocuments({ skillId: skill._id })).toBe(0); }); + it('findSkillBySourceIdentity searches only the requested tenant bucket', async () => { + const upstreamId = 'librechat-skills:skills/research'; + const sourceMetadata = { + provider: 'github', + sourceId: 'librechat-skills', + upstreamId, + }; + const author = new mongoose.Types.ObjectId(); + const tenantSkill = await Skill.create({ + name: 'research', + description: 'A tenant-scoped GitHub skill mirror.', + body: 'tenant body', + author, + authorName: 'GitHub Sync', + tenantId: 'tenant-a', + source: 'github', + sourceMetadata, + }); + const ambientSkill = await Skill.create({ + name: 'research', + description: 'An ambient GitHub skill mirror.', + body: 'ambient body', + author, + authorName: 'GitHub Sync', + source: 'github', + sourceMetadata, + }); + + const ambientResult = await methods.findSkillBySourceIdentity({ + source: 'github', + upstreamId, + }); + const tenantResult = await methods.findSkillBySourceIdentity({ + source: 'github', + upstreamId, + tenantId: 'tenant-a', + }); + + expect(ambientResult?._id.toString()).toBe(ambientSkill._id.toString()); + expect(tenantResult?._id.toString()).toBe(tenantSkill._id.toString()); + }); + it('listSkillsByAccess returns only accessible skills and paginates by cursor', async () => { const ids: mongoose.Types.ObjectId[] = []; for (let i = 0; i < 3; i++) { diff --git a/packages/data-schemas/src/methods/skill.ts b/packages/data-schemas/src/methods/skill.ts index 74273335c7..d5e72b9603 100644 --- a/packages/data-schemas/src/methods/skill.ts +++ b/packages/data-schemas/src/methods/skill.ts @@ -498,6 +498,8 @@ export type UpdateSkillInput = { frontmatter?: Record; category?: string; alwaysApply?: boolean; + source?: 'inline' | 'github' | 'notion'; + sourceMetadata?: Record; }; export type GetAuthorSkillByNameParams = { @@ -622,6 +624,7 @@ export type UpsertSkillFileInput = { storageKey?: string; storageRegion?: string; source: string; + sourceMetadata?: Record; mimeType: string; bytes: number; isExecutable?: boolean; @@ -902,6 +905,15 @@ export function createSkillMethods( }) => Promise; deleteSkill: (id: string) => Promise<{ deleted: boolean }>; deleteUserSkills: (userId: Types.ObjectId | string) => Promise; + findSkillBySourceIdentity: (params: { + source: 'github' | 'notion'; + upstreamId: string; + tenantId?: string; + }) => Promise<(ISkill & { _id: Types.ObjectId }) | null>; + listSkillsBySource: (params: { + source: 'github' | 'notion'; + sourceId: string; + }) => Promise>; listSkillFiles: ( skillId: Types.ObjectId | string, ) => Promise>; @@ -1349,6 +1361,8 @@ export function createSkillMethods( if (update.displayTitle !== undefined) setPayload.displayTitle = update.displayTitle; if (update.description !== undefined) setPayload.description = update.description; if (update.body !== undefined) setPayload.body = update.body; + if (update.source !== undefined) setPayload.source = update.source; + if (update.sourceMetadata !== undefined) setPayload.sourceMetadata = update.sourceMetadata; if (update.frontmatter !== undefined) { setPayload.frontmatter = update.frontmatter; /** @@ -1500,6 +1514,35 @@ export function createSkillMethods( return res.deletedCount ?? 0; } + async function findSkillBySourceIdentity(params: { + source: 'github' | 'notion'; + upstreamId: string; + tenantId?: string; + }): Promise<(ISkill & { _id: Types.ObjectId }) | null> { + const Skill = mongoose.models.Skill as Model; + const tenantFilter: FilterQuery = params.tenantId + ? { tenantId: params.tenantId } + : { $or: [{ tenantId: { $exists: false } }, { tenantId: null }] }; + const doc = await Skill.findOne({ + source: params.source, + 'sourceMetadata.upstreamId': params.upstreamId, + ...tenantFilter, + }).lean(); + return (doc as unknown as (ISkill & { _id: Types.ObjectId }) | null) ?? null; + } + + async function listSkillsBySource(params: { + source: 'github' | 'notion'; + sourceId: string; + }): Promise> { + const Skill = mongoose.models.Skill as Model; + const rows = await Skill.find({ + source: params.source, + 'sourceMetadata.sourceId': params.sourceId, + }).lean(); + return rows as unknown as Array; + } + /** * Atomically bumps `Skill.version` and adjusts `fileCount` by `delta`. * `delta` is `+1` when a new file is inserted, `-1` when one is deleted, and @@ -1568,6 +1611,7 @@ export function createSkillMethods( storageKey: row.storageKey, storageRegion: row.storageRegion, source: row.source, + sourceMetadata: row.sourceMetadata, mimeType: row.mimeType, bytes: row.bytes, category, @@ -1664,6 +1708,8 @@ export function createSkillMethods( updateSkill, deleteSkill, deleteUserSkills, + findSkillBySourceIdentity, + listSkillsBySource, listSkillFiles, upsertSkillFile, deleteSkillFile, diff --git a/packages/data-schemas/src/methods/skillSync.spec.ts b/packages/data-schemas/src/methods/skillSync.spec.ts new file mode 100644 index 0000000000..d384db1d7c --- /dev/null +++ b/packages/data-schemas/src/methods/skillSync.spec.ts @@ -0,0 +1,194 @@ +import mongoose from 'mongoose'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import type { ISkillSyncCredential } from '~/types/skillSync'; +import { createSkillSyncMethods } from './skillSync'; +import { encryptV2, decryptV2 } from '~/crypto'; +import { createModels } from '../models'; + +jest.mock('~/crypto', () => ({ + encryptV2: jest.fn(async (value: string) => `encrypted:${value}`), + decryptV2: jest.fn(async (value: string) => value.replace(/^encrypted:/, '')), +})); + +let mongoServer: MongoMemoryServer; +let methods: ReturnType; + +beforeAll(async () => { + mongoServer = await MongoMemoryServer.create(); + await mongoose.connect(mongoServer.getUri()); + createModels(mongoose); + methods = createSkillSyncMethods(mongoose); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongoServer.stop(); +}); + +beforeEach(async () => { + await mongoose.models.SkillSyncCredential.deleteMany({}); + await mongoose.models.SkillSyncStatus.deleteMany({}); + jest.clearAllMocks(); +}); + +describe('createSkillSyncMethods', () => { + it('encrypts GitHub tokens and never exposes stored token material in summaries', async () => { + const summary = await methods.upsertSkillSyncCredential({ + provider: 'github', + credentialKey: 'github-skills-prod', + token: 'ghp_secret', + }); + expect(encryptV2).toHaveBeenCalledWith('ghp_secret'); + expect(summary).toMatchObject({ + provider: 'github', + credentialKey: 'github-skills-prod', + credentialPresent: true, + }); + expect(JSON.stringify(summary)).not.toContain('ghp_secret'); + expect(JSON.stringify(summary)).not.toContain('encrypted:ghp_secret'); + + const raw = (await mongoose.models.SkillSyncCredential.findOne({ + provider: 'github', + credentialKey: 'github-skills-prod', + }) + .select('+encryptedToken +tokenHash') + .lean()) as ISkillSyncCredential | null; + if (!raw) { + throw new Error('Expected stored skill sync credential'); + } + expect(raw.encryptedToken).toBe('encrypted:ghp_secret'); + expect(raw.tokenHash).toHaveLength(64); + }); + + it('decrypts GitHub tokens only through the token accessor', async () => { + await methods.upsertSkillSyncCredential({ + provider: 'github', + credentialKey: 'github-skills-prod', + token: 'github_pat_secret', + }); + const token = await methods.getSkillSyncCredentialToken('github', 'github-skills-prod'); + expect(decryptV2).toHaveBeenCalledWith('encrypted:github_pat_secret'); + expect(token).toBe('github_pat_secret'); + }); + + it('uses the status collection as a Mongo-backed sync lock', async () => { + await expect( + methods.tryAcquireSkillSyncLock({ + provider: 'github', + lockOwner: 'worker-a', + leaseMs: 60_000, + }), + ).resolves.toBe(true); + await expect( + methods.tryAcquireSkillSyncLock({ + provider: 'github', + lockOwner: 'worker-a', + leaseMs: 60_000, + }), + ).resolves.toBe(false); + await expect( + methods.tryAcquireSkillSyncLock({ + provider: 'github', + lockOwner: 'worker-b', + leaseMs: 60_000, + }), + ).resolves.toBe(false); + await expect( + methods.refreshSkillSyncLock({ + provider: 'github', + lockOwner: 'worker-a', + leaseMs: 60_000, + }), + ).resolves.toBe(true); + await methods.releaseSkillSyncLock({ provider: 'github', lockOwner: 'worker-a' }); + await expect( + methods.tryAcquireSkillSyncLock({ + provider: 'github', + lockOwner: 'worker-b', + leaseMs: 60_000, + }), + ).resolves.toBe(true); + }); + + it('clears stale source errors after a successful sync status update', async () => { + await methods.upsertSkillSyncStatus({ + provider: 'github', + sourceId: 'librechat-skills', + status: 'failed', + errorCode: 'SKILL_PARSE_FAILED', + errorMessage: 'bad frontmatter', + }); + + const success = await methods.upsertSkillSyncStatus({ + provider: 'github', + sourceId: 'librechat-skills', + status: 'succeeded', + syncedSkillCount: 1, + syncedFileCount: 2, + }); + + expect(success.status).toBe('succeeded'); + expect(success.errorCode).toBeUndefined(); + expect(success.errorMessage).toBeUndefined(); + }); + + it('keeps status rows separate for the same source id in different tenants', async () => { + await methods.upsertSkillSyncStatus({ + provider: 'github', + sourceId: 'shared-source', + tenantId: 'tenant-a', + status: 'succeeded', + syncedSkillCount: 1, + }); + await methods.upsertSkillSyncStatus({ + provider: 'github', + sourceId: 'shared-source', + tenantId: 'tenant-b', + status: 'failed', + errorCode: 'TENANT_B_FAILURE', + }); + + const tenantA = await methods.getSkillSyncStatus('github', 'shared-source', 'tenant-a'); + const tenantB = await methods.getSkillSyncStatus('github', 'shared-source', 'tenant-b'); + + expect(tenantA).toMatchObject({ + sourceId: 'shared-source', + tenantId: 'tenant-a', + status: 'succeeded', + syncedSkillCount: 1, + }); + expect(tenantB).toMatchObject({ + sourceId: 'shared-source', + tenantId: 'tenant-b', + status: 'failed', + errorCode: 'TENANT_B_FAILURE', + }); + }); + + it('keeps sync locks separate per tenant', async () => { + await expect( + methods.tryAcquireSkillSyncLock({ + provider: 'github', + tenantId: 'tenant-a', + lockOwner: 'worker-a', + leaseMs: 60_000, + }), + ).resolves.toBe(true); + await expect( + methods.tryAcquireSkillSyncLock({ + provider: 'github', + tenantId: 'tenant-b', + lockOwner: 'worker-b', + leaseMs: 60_000, + }), + ).resolves.toBe(true); + await expect( + methods.tryAcquireSkillSyncLock({ + provider: 'github', + tenantId: 'tenant-a', + lockOwner: 'worker-c', + leaseMs: 60_000, + }), + ).resolves.toBe(false); + }); +}); diff --git a/packages/data-schemas/src/methods/skillSync.ts b/packages/data-schemas/src/methods/skillSync.ts new file mode 100644 index 0000000000..87c749f400 --- /dev/null +++ b/packages/data-schemas/src/methods/skillSync.ts @@ -0,0 +1,382 @@ +import { createHash } from 'crypto'; +import type { Model, Types } from 'mongoose'; +import type { + ISkillSyncStatus, + SkillSyncProvider, + SkillSyncRunStatus, + ISkillSyncStatusDocument, + ISkillSyncCredential, + ISkillSyncCredentialDocument, +} from '~/types/skillSync'; +import { encryptV2, decryptV2 } from '~/crypto'; + +const LOCK_SOURCE_ID = '__global_lock__'; + +export type SkillSyncCredentialSummary = { + provider: SkillSyncProvider; + credentialKey: string; + credentialPresent: boolean; + tokenFingerprint?: string; + createdAt?: Date; + updatedAt?: Date; +}; + +export type UpsertSkillSyncCredentialInput = { + provider: SkillSyncProvider; + credentialKey: string; + token: string; + userId?: Types.ObjectId; +}; + +export type SkillSyncStatusInput = { + provider: SkillSyncProvider; + sourceId: string; + tenantId?: string; + status: SkillSyncRunStatus; + credentialKey?: string; + owner?: string; + repo?: string; + ref?: string; + paths?: string[]; + startedAt?: Date; + finishedAt?: Date; + errorCode?: string; + errorMessage?: string; + syncedSkillCount?: number; + syncedFileCount?: number; + deletedSkillCount?: number; + deletedFileCount?: number; +}; + +export type SkillSyncLockInput = { + provider: SkillSyncProvider; + lockOwner: string; + leaseMs: number; + tenantId?: string; +}; + +export type SkillSyncReleaseLockInput = { + provider: SkillSyncProvider; + lockOwner: string; + tenantId?: string; +}; + +export type SkillSyncMethods = { + upsertSkillSyncCredential: ( + input: UpsertSkillSyncCredentialInput, + ) => Promise; + deleteSkillSyncCredential: ( + provider: SkillSyncProvider, + credentialKey: string, + ) => Promise<{ deleted: boolean }>; + listSkillSyncCredentials: (provider: SkillSyncProvider) => Promise; + getSkillSyncCredentialToken: ( + provider: SkillSyncProvider, + credentialKey: string, + ) => Promise; + getSkillSyncCredentialSummary: ( + provider: SkillSyncProvider, + credentialKey: string, + ) => Promise; + listSkillSyncStatuses: (provider: SkillSyncProvider) => Promise; + getSkillSyncStatus: ( + provider: SkillSyncProvider, + sourceId: string, + tenantId?: string, + ) => Promise; + upsertSkillSyncStatus: (input: SkillSyncStatusInput) => Promise; + tryAcquireSkillSyncLock: (params: SkillSyncLockInput) => Promise; + refreshSkillSyncLock: (params: SkillSyncLockInput) => Promise; + releaseSkillSyncLock: (params: SkillSyncReleaseLockInput) => Promise; +}; + +function hashToken(token: string): string { + return createHash('sha256').update(token).digest('hex'); +} + +function tenantStatusCondition(tenantId?: string) { + return tenantId ? { tenantId } : { tenantId: { $exists: false } }; +} + +function summarizeCredential( + credential: Pick< + ISkillSyncCredential, + 'provider' | 'credentialKey' | 'tokenHash' | 'createdAt' | 'updatedAt' + >, +): SkillSyncCredentialSummary { + return { + provider: credential.provider, + credentialKey: credential.credentialKey, + credentialPresent: true, + tokenFingerprint: credential.tokenHash.slice(0, 12), + createdAt: credential.createdAt, + updatedAt: credential.updatedAt, + }; +} + +export function createSkillSyncMethods(mongoose: typeof import('mongoose')): SkillSyncMethods { + async function upsertSkillSyncCredential( + input: UpsertSkillSyncCredentialInput, + ): Promise { + const Credential = mongoose.models.SkillSyncCredential as Model; + const encryptedToken = await encryptV2(input.token); + const tokenHash = hashToken(input.token); + const update = { + $set: { + encryptedToken, + tokenHash, + updatedBy: input.userId, + }, + $setOnInsert: { + provider: input.provider, + credentialKey: input.credentialKey, + createdBy: input.userId, + }, + }; + const credential = await Credential.findOneAndUpdate( + { provider: input.provider, credentialKey: input.credentialKey }, + update, + { upsert: true, new: true, setDefaultsOnInsert: true }, + ) + .select('+tokenHash') + .lean(); + return summarizeCredential(credential as ISkillSyncCredential); + } + + async function deleteSkillSyncCredential( + provider: SkillSyncProvider, + credentialKey: string, + ): Promise<{ deleted: boolean }> { + const Credential = mongoose.models.SkillSyncCredential as Model; + const result = await Credential.deleteOne({ provider, credentialKey }); + return { deleted: (result.deletedCount ?? 0) > 0 }; + } + + async function listSkillSyncCredentials( + provider: SkillSyncProvider, + ): Promise { + const Credential = mongoose.models.SkillSyncCredential as Model; + const rows = await Credential.find({ provider }) + .select('+tokenHash') + .sort({ credentialKey: 1 }) + .lean(); + return rows.map((row) => summarizeCredential(row as ISkillSyncCredential)); + } + + async function getSkillSyncCredentialToken( + provider: SkillSyncProvider, + credentialKey: string, + ): Promise { + const Credential = mongoose.models.SkillSyncCredential as Model; + const credential = await Credential.findOne({ provider, credentialKey }) + .select('+encryptedToken') + .lean(); + if (!credential) { + return null; + } + return decryptV2((credential as ISkillSyncCredential).encryptedToken); + } + + async function getSkillSyncCredentialSummary( + provider: SkillSyncProvider, + credentialKey: string, + ): Promise { + const Credential = mongoose.models.SkillSyncCredential as Model; + const credential = await Credential.findOne({ provider, credentialKey }) + .select('+tokenHash') + .lean(); + if (!credential) { + return null; + } + return summarizeCredential(credential as ISkillSyncCredential); + } + + async function listSkillSyncStatuses(provider: SkillSyncProvider): Promise { + const Status = mongoose.models.SkillSyncStatus as Model; + const rows = await Status.find({ provider, sourceId: { $ne: LOCK_SOURCE_ID } }) + .sort({ sourceId: 1 }) + .lean(); + return rows; + } + + async function getSkillSyncStatus( + provider: SkillSyncProvider, + sourceId: string, + tenantId?: string, + ): Promise { + const Status = mongoose.models.SkillSyncStatus as Model; + const row = await Status.findOne({ + provider, + sourceId, + ...tenantStatusCondition(tenantId), + }).lean(); + return row ?? null; + } + + async function upsertSkillSyncStatus(input: SkillSyncStatusInput): Promise { + const Status = mongoose.models.SkillSyncStatus as Model; + const now = new Date(); + const success = input.status === 'succeeded'; + const failure = input.status === 'failed'; + const setPayload: Partial = { + status: input.status, + credentialKey: input.credentialKey, + owner: input.owner, + repo: input.repo, + ref: input.ref, + paths: input.paths, + startedAt: input.startedAt, + finishedAt: input.finishedAt, + syncedSkillCount: input.syncedSkillCount ?? 0, + syncedFileCount: input.syncedFileCount ?? 0, + deletedSkillCount: input.deletedSkillCount ?? 0, + deletedFileCount: input.deletedFileCount ?? 0, + ...(success ? { lastSuccessAt: input.finishedAt ?? now } : {}), + ...(failure ? { lastFailureAt: input.finishedAt ?? now } : {}), + }; + if (failure) { + setPayload.errorCode = input.errorCode; + setPayload.errorMessage = input.errorMessage; + } + const unsetPayload = failure ? {} : { errorCode: '', errorMessage: '' }; + const row = await Status.findOneAndUpdate( + { + provider: input.provider, + sourceId: input.sourceId, + ...tenantStatusCondition(input.tenantId), + }, + { + $set: setPayload, + ...(failure ? {} : { $unset: unsetPayload }), + $setOnInsert: { + provider: input.provider, + sourceId: input.sourceId, + ...(input.tenantId ? { tenantId: input.tenantId } : {}), + }, + }, + { upsert: true, new: true, setDefaultsOnInsert: true }, + ).lean(); + return row; + } + + async function tryAcquireSkillSyncLock(params: SkillSyncLockInput): Promise { + const Status = mongoose.models.SkillSyncStatus as Model; + const now = new Date(); + const lockExpiresAt = new Date(now.getTime() + params.leaseMs); + const existing = await Status.findOne({ + provider: params.provider, + sourceId: LOCK_SOURCE_ID, + ...tenantStatusCondition(params.tenantId), + }).lean(); + if (existing?.lockOwner && existing.lockExpiresAt && existing.lockExpiresAt > now) { + return false; + } + if (!existing) { + try { + await Status.create({ + provider: params.provider, + sourceId: LOCK_SOURCE_ID, + ...(params.tenantId ? { tenantId: params.tenantId } : {}), + status: 'running', + lockOwner: params.lockOwner, + lockExpiresAt, + startedAt: now, + }); + return true; + } catch (error) { + if ((error as { code?: number }).code === 11000) { + return false; + } + throw error; + } + } + try { + const row = await Status.findOneAndUpdate( + { + provider: params.provider, + sourceId: LOCK_SOURCE_ID, + ...tenantStatusCondition(params.tenantId), + $or: [{ lockExpiresAt: { $exists: false } }, { lockExpiresAt: { $lte: now } }], + }, + { + $set: { + status: 'running', + lockOwner: params.lockOwner, + lockExpiresAt, + startedAt: now, + }, + $setOnInsert: { + provider: params.provider, + sourceId: LOCK_SOURCE_ID, + ...(params.tenantId ? { tenantId: params.tenantId } : {}), + }, + }, + { upsert: true, new: true, setDefaultsOnInsert: true }, + ).lean(); + return (row as ISkillSyncStatus | null)?.lockOwner === params.lockOwner; + } catch (error) { + if ((error as { code?: number }).code === 11000) { + return false; + } + throw error; + } + } + + async function refreshSkillSyncLock(params: SkillSyncLockInput): Promise { + const Status = mongoose.models.SkillSyncStatus as Model; + const now = new Date(); + const row = await Status.findOneAndUpdate( + { + provider: params.provider, + sourceId: LOCK_SOURCE_ID, + ...tenantStatusCondition(params.tenantId), + lockOwner: params.lockOwner, + lockExpiresAt: { $gt: now }, + }, + { + $set: { + status: 'running', + lockExpiresAt: new Date(now.getTime() + params.leaseMs), + }, + }, + { new: true }, + ).lean(); + return Boolean(row); + } + + async function releaseSkillSyncLock(params: SkillSyncReleaseLockInput): Promise { + const Status = mongoose.models.SkillSyncStatus as Model; + await Status.updateOne( + { + provider: params.provider, + sourceId: LOCK_SOURCE_ID, + ...tenantStatusCondition(params.tenantId), + lockOwner: params.lockOwner, + }, + { + $set: { + status: 'idle', + finishedAt: new Date(), + }, + $unset: { + lockOwner: '', + lockExpiresAt: '', + }, + }, + ); + } + + return { + upsertSkillSyncCredential, + deleteSkillSyncCredential, + listSkillSyncCredentials, + getSkillSyncCredentialToken, + getSkillSyncCredentialSummary, + listSkillSyncStatuses, + getSkillSyncStatus, + upsertSkillSyncStatus, + tryAcquireSkillSyncLock, + refreshSkillSyncLock, + releaseSkillSyncLock, + }; +} diff --git a/packages/data-schemas/src/migrations/tenantIndexes.spec.ts b/packages/data-schemas/src/migrations/tenantIndexes.spec.ts index a2fb2fef01..986adb5ab8 100644 --- a/packages/data-schemas/src/migrations/tenantIndexes.spec.ts +++ b/packages/data-schemas/src/migrations/tenantIndexes.spec.ts @@ -113,6 +113,11 @@ describe('dropSupersededTenantIndexes', () => { { idOnTheSource: 1, source: 1 }, { unique: true, name: 'idOnTheSource_1_source_1' }, ); + + await db.createCollection('skillsyncstatuses'); + await db + .collection('skillsyncstatuses') + .createIndex({ provider: 1, sourceId: 1 }, { unique: true, name: 'provider_1_sourceId_1' }); }); it('drops all superseded indexes', async () => { @@ -160,6 +165,13 @@ describe('dropSupersededTenantIndexes', () => { expect(indexNames).not.toContain('conversationId_1_user_1'); }); + + it('old skill sync status unique index is gone', async () => { + const indexes = await mongoose.connection.db!.collection('skillsyncstatuses').indexes(); + const indexNames = indexes.map((idx) => idx.name); + + expect(indexNames).not.toContain('provider_1_sourceId_1'); + }); }); describe('multi-tenant writes after migration', () => { @@ -287,6 +299,7 @@ describe('dropSupersededTenantIndexes', () => { 'mcpservers', 'files', 'groups', + 'skillsyncstatuses', ]; for (const col of expectedCollections) { diff --git a/packages/data-schemas/src/migrations/tenantIndexes.ts b/packages/data-schemas/src/migrations/tenantIndexes.ts index e511bb1736..9a84997c5c 100644 --- a/packages/data-schemas/src/migrations/tenantIndexes.ts +++ b/packages/data-schemas/src/migrations/tenantIndexes.ts @@ -35,6 +35,7 @@ const SUPERSEDED_INDEXES: Record = { mcpservers: ['serverName_1'], files: ['filename_1_conversationId_1_context_1'], groups: ['idOnTheSource_1_source_1'], + skillsyncstatuses: ['provider_1_sourceId_1'], }; interface MigrationResult { diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts index c8a9808bd9..476091ec9c 100644 --- a/packages/data-schemas/src/models/index.ts +++ b/packages/data-schemas/src/models/index.ts @@ -1,3 +1,5 @@ +import { createSkillSyncCredentialModel } from './skillSyncCredential'; +import { createSkillSyncStatusModel } from './skillSyncStatus'; import { createConversationTagModel } from './conversationTag'; import { createAgentCategoryModel } from './agentCategory'; import { createChatProjectModel } from './chatProject'; @@ -60,6 +62,8 @@ export function createModels(mongoose: typeof import('mongoose')): { PromptGroup: ReturnType; Skill: ReturnType; SkillFile: ReturnType; + SkillSyncCredential: ReturnType; + SkillSyncStatus: ReturnType; ConversationTag: ReturnType; SharedLink: ReturnType; ToolCall: ReturnType; @@ -95,6 +99,8 @@ export function createModels(mongoose: typeof import('mongoose')): { PromptGroup: createPromptGroupModel(mongoose), Skill: createSkillModel(mongoose), SkillFile: createSkillFileModel(mongoose), + SkillSyncCredential: createSkillSyncCredentialModel(mongoose), + SkillSyncStatus: createSkillSyncStatusModel(mongoose), ConversationTag: createConversationTagModel(mongoose), SharedLink: createSharedLinkModel(mongoose), ToolCall: createToolCallModel(mongoose), diff --git a/packages/data-schemas/src/models/plugins/tenantIsolation.coverage.spec.ts b/packages/data-schemas/src/models/plugins/tenantIsolation.coverage.spec.ts index 13b7af41c2..61708666f1 100644 --- a/packages/data-schemas/src/models/plugins/tenantIsolation.coverage.spec.ts +++ b/packages/data-schemas/src/models/plugins/tenantIsolation.coverage.spec.ts @@ -11,10 +11,12 @@ const TENANT_ISOLATION_APPLIED = Symbol.for('librechat:tenantIsolation'); /** * Models that carry a `tenantId` field but intentionally do NOT use the * tenant-isolation plugin. SystemGrant scopes tenancy manually inside its - * methods (see models/systemGrant). Adding an entry here must be a deliberate, - * reviewed decision — that is the whole point of this guard. + * methods (see models/systemGrant). SkillSyncStatus stores both app-wide YAML + * status rows and tenant-scoped override rows, so its methods apply explicit + * tenant filters instead of ambient ALS scoping. Adding an entry here must be a + * deliberate, reviewed decision — that is the whole point of this guard. */ -const MANUAL_TENANT_SCOPING = new Set(['SystemGrant']); +const MANUAL_TENANT_SCOPING = new Set(['SystemGrant', 'SkillSyncStatus']); function isPluginApplied(schema: mongoose.Schema): boolean { return (schema as unknown as { [key: symbol]: boolean })[TENANT_ISOLATION_APPLIED] === true; diff --git a/packages/data-schemas/src/models/skillSyncCredential.ts b/packages/data-schemas/src/models/skillSyncCredential.ts new file mode 100644 index 0000000000..4e0e636fb6 --- /dev/null +++ b/packages/data-schemas/src/models/skillSyncCredential.ts @@ -0,0 +1,14 @@ +import { Model } from 'mongoose'; +import type { ISkillSyncCredentialDocument } from '~/types/skillSync'; +import skillSyncCredentialSchema from '~/schema/skillSyncCredential'; + +export function createSkillSyncCredentialModel( + mongoose: typeof import('mongoose'), +): Model { + // GitHub skill sync is intentionally app-wide in v1; credentials are referenced by + // admin-managed config keys and are never returned by tenant-scoped APIs. + return ( + mongoose.models.SkillSyncCredential || + mongoose.model('SkillSyncCredential', skillSyncCredentialSchema) + ); +} diff --git a/packages/data-schemas/src/models/skillSyncStatus.ts b/packages/data-schemas/src/models/skillSyncStatus.ts new file mode 100644 index 0000000000..9b15c65ae6 --- /dev/null +++ b/packages/data-schemas/src/models/skillSyncStatus.ts @@ -0,0 +1,14 @@ +import { Model } from 'mongoose'; +import type { ISkillSyncStatusDocument } from '~/types/skillSync'; +import skillSyncStatusSchema from '~/schema/skillSyncStatus'; + +export function createSkillSyncStatusModel( + mongoose: typeof import('mongoose'), +): Model { + // GitHub skill sync status supports app-wide YAML sources and tenant-scoped + // resolved config sources from admin overrides. + return ( + mongoose.models.SkillSyncStatus || + mongoose.model('SkillSyncStatus', skillSyncStatusSchema) + ); +} diff --git a/packages/data-schemas/src/schema/index.ts b/packages/data-schemas/src/schema/index.ts index 3a796077cb..0dfd4dcb30 100644 --- a/packages/data-schemas/src/schema/index.ts +++ b/packages/data-schemas/src/schema/index.ts @@ -19,6 +19,8 @@ export { default as promptGroupSchema } from './promptGroup'; export { default as roleSchema } from './role'; export { default as sessionSchema } from './session'; export { default as shareSchema } from './share'; +export { default as skillSyncCredentialSchema } from './skillSyncCredential'; +export { default as skillSyncStatusSchema } from './skillSyncStatus'; export { default as tokenSchema } from './token'; export { default as toolCallSchema } from './toolCall'; export { default as transactionSchema } from './transaction'; diff --git a/packages/data-schemas/src/schema/skill.ts b/packages/data-schemas/src/schema/skill.ts index d21839a2db..9ece6d82cc 100644 --- a/packages/data-schemas/src/schema/skill.ts +++ b/packages/data-schemas/src/schema/skill.ts @@ -182,12 +182,9 @@ const skillSchema: Schema = new Schema( /** * Provenance of this skill's canonical definition. * - * - `inline` — authored directly inside LibreChat (the only path wired - * up in phase 1). - * - `github` / `notion` — **reserved for phase 2+ external sync**. No - * code path currently produces these values. The column exists now so - * a future sync worker can populate `source` + `sourceMetadata` without - * a schema migration. + * - `inline` — authored directly inside LibreChat. + * - `github` — mirrored from a configured GitHub skill sync source. + * - `notion` — reserved for future external sync integrations. */ source: { type: String, @@ -195,10 +192,8 @@ const skillSchema: Schema = new Schema( default: 'inline', }, /** - * Arbitrary JSON provenance payload keyed by `source`. Phase 2+ sync - * workers will use this to store the upstream commit SHA (github), - * page id (notion), etc. Unused in phase 1 — kept `Mixed` to avoid - * committing to a shape before the sync paths exist. + * Arbitrary JSON provenance payload keyed by `source`. GitHub sync stores + * source id, upstream path, commit/blob SHAs, and sync status here. */ sourceMetadata: { type: Schema.Types.Mixed, @@ -234,5 +229,7 @@ skillSchema.index({ author: 1, tenantId: 1 }); skillSchema.index({ category: 1, updatedAt: -1 }); skillSchema.index({ updatedAt: -1, _id: 1 }); skillSchema.index({ name: 1, author: 1, tenantId: 1 }, { unique: true }); +skillSchema.index({ source: 1, 'sourceMetadata.upstreamId': 1, tenantId: 1 }); +skillSchema.index({ source: 1, 'sourceMetadata.sourceId': 1, tenantId: 1 }); export default skillSchema; diff --git a/packages/data-schemas/src/schema/skillFile.ts b/packages/data-schemas/src/schema/skillFile.ts index efb5a29c1d..27f38847b9 100644 --- a/packages/data-schemas/src/schema/skillFile.ts +++ b/packages/data-schemas/src/schema/skillFile.ts @@ -69,6 +69,9 @@ const skillFileSchema: Schema = new Schema( type: String, required: true, }, + sourceMetadata: { + type: Schema.Types.Mixed, + }, mimeType: { type: String, required: true, diff --git a/packages/data-schemas/src/schema/skillSyncCredential.ts b/packages/data-schemas/src/schema/skillSyncCredential.ts new file mode 100644 index 0000000000..3ff90a1d79 --- /dev/null +++ b/packages/data-schemas/src/schema/skillSyncCredential.ts @@ -0,0 +1,45 @@ +import { Schema } from 'mongoose'; +import type { ISkillSyncCredentialDocument } from '~/types/skillSync'; + +const skillSyncCredentialSchema: Schema = new Schema( + { + provider: { + type: String, + enum: ['github'], + required: true, + index: true, + }, + credentialKey: { + type: String, + required: true, + maxlength: 64, + match: /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, + index: true, + }, + encryptedToken: { + type: String, + required: true, + select: false, + }, + tokenHash: { + type: String, + required: true, + select: false, + }, + createdBy: { + type: Schema.Types.ObjectId, + ref: 'User', + }, + updatedBy: { + type: Schema.Types.ObjectId, + ref: 'User', + }, + }, + { + timestamps: true, + }, +); + +skillSyncCredentialSchema.index({ provider: 1, credentialKey: 1 }, { unique: true }); + +export default skillSyncCredentialSchema; diff --git a/packages/data-schemas/src/schema/skillSyncStatus.ts b/packages/data-schemas/src/schema/skillSyncStatus.ts new file mode 100644 index 0000000000..0413f9d02e --- /dev/null +++ b/packages/data-schemas/src/schema/skillSyncStatus.ts @@ -0,0 +1,97 @@ +import { Schema } from 'mongoose'; +import type { ISkillSyncStatusDocument } from '~/types/skillSync'; + +const skillSyncStatusSchema: Schema = new Schema( + { + provider: { + type: String, + enum: ['github'], + required: true, + index: true, + }, + sourceId: { + type: String, + required: true, + maxlength: 128, + index: true, + }, + tenantId: { + type: String, + index: true, + }, + status: { + type: String, + enum: ['idle', 'running', 'succeeded', 'failed', 'skipped'], + default: 'idle', + required: true, + }, + credentialKey: { + type: String, + }, + owner: { + type: String, + }, + repo: { + type: String, + }, + ref: { + type: String, + }, + paths: { + type: [String], + default: undefined, + }, + startedAt: { + type: Date, + }, + finishedAt: { + type: Date, + }, + lastSuccessAt: { + type: Date, + }, + lastFailureAt: { + type: Date, + }, + errorCode: { + type: String, + }, + errorMessage: { + type: String, + }, + syncedSkillCount: { + type: Number, + default: 0, + min: 0, + }, + syncedFileCount: { + type: Number, + default: 0, + min: 0, + }, + deletedSkillCount: { + type: Number, + default: 0, + min: 0, + }, + deletedFileCount: { + type: Number, + default: 0, + min: 0, + }, + lockOwner: { + type: String, + }, + lockExpiresAt: { + type: Date, + index: true, + }, + }, + { + timestamps: true, + }, +); + +skillSyncStatusSchema.index({ provider: 1, sourceId: 1, tenantId: 1 }, { unique: true }); + +export default skillSyncStatusSchema; diff --git a/packages/data-schemas/src/types/app.ts b/packages/data-schemas/src/types/app.ts index 2b47e6f011..0eb0f4d60b 100644 --- a/packages/data-schemas/src/types/app.ts +++ b/packages/data-schemas/src/types/app.ts @@ -13,6 +13,7 @@ import type { TAssistantEndpoint, TAnthropicEndpoint, SummarizationConfig, + SkillSyncConfig, } from 'librechat-data-provider'; export type JsonSchemaType = { @@ -64,6 +65,8 @@ export interface AppConfig { webSearch?: TCustomConfig['webSearch']; /** Message filter configuration (PII and future filter types) */ messageFilter?: TCustomConfig['messageFilter']; + /** Skill sync configuration */ + skillSync?: SkillSyncConfig; /** File storage strategy ('local', 's3', 'firebase', 'azure_blob', 'cloudfront') */ fileStrategy: FileStorage; /** File strategies configuration */ diff --git a/packages/data-schemas/src/types/index.ts b/packages/data-schemas/src/types/index.ts index e00a016a64..da4f42c435 100644 --- a/packages/data-schemas/src/types/index.ts +++ b/packages/data-schemas/src/types/index.ts @@ -26,6 +26,7 @@ export * from './memory'; export * from './prompts'; /* Skills */ export * from './skill'; +export * from './skillSync'; /* Access Control */ export * from './accessRole'; export * from './aclEntry'; diff --git a/packages/data-schemas/src/types/skill.ts b/packages/data-schemas/src/types/skill.ts index a2cba928d1..e10c5bfda7 100644 --- a/packages/data-schemas/src/types/skill.ts +++ b/packages/data-schemas/src/types/skill.ts @@ -70,14 +70,14 @@ export interface ISkill { version: number; /** * Provenance of this skill's canonical definition. - * - `inline` — authored inside LibreChat (the only value phase 1 produces). - * - `github` / `notion` — reserved for phase 2+ external sync. Kept in the - * enum so a future sync worker can populate it without a migration. + * - `inline` — authored inside LibreChat. + * - `github` — mirrored from a configured GitHub skill sync source. + * - `notion` — reserved for future external sync integrations. */ source: 'inline' | 'github' | 'notion'; /** - * Provenance payload keyed by `source`. Phase 2+ sync workers will store - * upstream identifiers (commit SHA, page id, etc.) here. Unused in phase 1. + * Provenance payload keyed by `source`, including upstream identifiers + * such as GitHub source id, path, and commit/blob SHAs. */ sourceMetadata?: Record; /** Denormalized count of associated `SkillFile` rows. Kept in sync by skill methods. */ @@ -120,6 +120,7 @@ export interface ISkillFile { storageKey?: string; storageRegion?: string; source: string; + sourceMetadata?: Record; mimeType: string; bytes: number; category: 'script' | 'reference' | 'asset' | 'other'; diff --git a/packages/data-schemas/src/types/skillSync.ts b/packages/data-schemas/src/types/skillSync.ts new file mode 100644 index 0000000000..edd3bf5704 --- /dev/null +++ b/packages/data-schemas/src/types/skillSync.ts @@ -0,0 +1,45 @@ +import type { Document, Types } from 'mongoose'; + +export type SkillSyncProvider = 'github'; +export type SkillSyncRunStatus = 'idle' | 'running' | 'succeeded' | 'failed' | 'skipped'; + +export interface ISkillSyncCredential { + provider: SkillSyncProvider; + credentialKey: string; + encryptedToken: string; + tokenHash: string; + createdBy?: Types.ObjectId; + updatedBy?: Types.ObjectId; + createdAt?: Date; + updatedAt?: Date; +} + +export interface ISkillSyncCredentialDocument extends ISkillSyncCredential, Document {} + +export interface ISkillSyncStatus { + provider: SkillSyncProvider; + sourceId: string; + tenantId?: string; + status: SkillSyncRunStatus; + credentialKey?: string; + owner?: string; + repo?: string; + ref?: string; + paths?: string[]; + startedAt?: Date; + finishedAt?: Date; + lastSuccessAt?: Date; + lastFailureAt?: Date; + errorCode?: string; + errorMessage?: string; + syncedSkillCount: number; + syncedFileCount: number; + deletedSkillCount: number; + deletedFileCount: number; + lockOwner?: string; + lockExpiresAt?: Date; + createdAt?: Date; + updatedAt?: Date; +} + +export interface ISkillSyncStatusDocument extends ISkillSyncStatus, Document {}