From 2c8d54e18c7a132019529062f5837a32fe783ac5 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 5 Jun 2026 10:24:28 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=82=EF=B8=8F=20feat:=20Add=20Deploymen?= =?UTF-8?q?t=20Skill=20Directory=20(#13523)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add deployment skill directory * chore: Address deployment skill review feedback * fix: Include deployment skill file metadata * test: Add deployment skills e2e smoke test --- .env.example | 4 + Dockerfile | 2 +- Dockerfile.multi | 1 + .../controllers/SkillStatesController.js | 23 +- .../agents/__tests__/openai.spec.js | 2 + .../agents/__tests__/responses.unit.spec.js | 2 + api/server/controllers/agents/openai.js | 23 +- api/server/controllers/agents/responses.js | 23 +- api/server/index.js | 2 + .../accessResources/canAccessSkillResource.js | 28 +- api/server/routes/skills.js | 37 +- .../services/Endpoints/agents/initialize.js | 35 +- .../services/Endpoints/agents/skillDeps.js | 47 +- .../Endpoints/agents/skillDeps.spec.js | 5 + deploy-compose.yml | 1 + docker-compose.yml | 1 + .../e2e-deployment-skill/SKILL.md | 10 + .../e2e-deployment-skill/guide.txt | 1 + e2e/playwright.config.mock.ts | 2 + e2e/specs/mock/deployment-skills.spec.ts | 218 ++++ .../api/src/agents/__tests__/skills.test.ts | 57 + packages/api/src/agents/handlers.ts | 2 + packages/api/src/agents/initialize.ts | 6 + packages/api/src/agents/skills.ts | 11 +- .../src/skills/__tests__/deployment.test.ts | 327 ++++++ packages/api/src/skills/deployment.ts | 983 ++++++++++++++++++ packages/api/src/skills/import.ts | 2 +- packages/api/src/skills/index.ts | 1 + packages/data-provider/src/types/skills.ts | 8 +- packages/data-schemas/src/index.ts | 8 + packages/data-schemas/src/methods/index.ts | 18 + skill/README.md | 15 + 32 files changed, 1836 insertions(+), 69 deletions(-) create mode 100644 e2e/fixtures/deployment-skills/e2e-deployment-skill/SKILL.md create mode 100644 e2e/fixtures/deployment-skills/e2e-deployment-skill/guide.txt create mode 100644 e2e/specs/mock/deployment-skills.spec.ts create mode 100644 packages/api/src/skills/__tests__/deployment.test.ts create mode 100644 packages/api/src/skills/deployment.ts create mode 100644 skill/README.md diff --git a/.env.example b/.env.example index 348348f99a..1326c4d9e5 100644 --- a/.env.example +++ b/.env.example @@ -108,6 +108,10 @@ NODE_MAX_OLD_SPACE_SIZE=6144 # CONFIG_PATH="/alternative/path/to/librechat.yaml" +# Deployment skills are loaded read-only at startup and exposed to all users +# with the Skills capability enabled. Defaults to project root ./skill. +# DEPLOYMENT_SKILLS_DIR=./skill + #==================# # Langfuse Tracing # #==================# diff --git a/Dockerfile b/Dockerfile index 6253913262..1761e8e1e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN \ # Allow mounting of these files, which have no default touch .env ; \ # Create directories for the volumes to inherit the correct permissions - mkdir -p /app/client/public/images /app/logs /app/uploads ; \ + mkdir -p /app/client/public/images /app/logs /app/uploads /app/skill ; \ npm config set fetch-retry-maxtimeout 600000 ; \ npm config set fetch-retries 5 ; \ npm config set fetch-retry-mintimeout 15000 ; \ diff --git a/Dockerfile.multi b/Dockerfile.multi index 745e54b8a4..754f8acb78 100644 --- a/Dockerfile.multi +++ b/Dockerfile.multi @@ -109,6 +109,7 @@ RUN attempt=1; \ done COPY api ./api COPY config ./config +COPY skill ./skill COPY --from=data-provider-build /app/packages/data-provider/dist ./packages/data-provider/dist COPY --from=data-schemas-build /app/packages/data-schemas/dist ./packages/data-schemas/dist COPY --from=api-package-build /app/packages/api/dist ./packages/api/dist diff --git a/api/server/controllers/SkillStatesController.js b/api/server/controllers/SkillStatesController.js index 35679d2ac6..1afeb27141 100644 --- a/api/server/controllers/SkillStatesController.js +++ b/api/server/controllers/SkillStatesController.js @@ -5,6 +5,8 @@ const { toSkillStatesRecord, validateSkillStatesPayload, pruneOrphanSkillStates, + getDeploymentSkillIds, + mergeDeploymentSkillIds, } = require('@librechat/api'); const { ResourceType, PermissionBits } = require('librechat-data-provider'); const { findAccessibleResources } = require('~/server/services/PermissionService'); @@ -21,15 +23,20 @@ function buildPruneDeps(user) { const existing = await Skill.find({ _id: { $in: validIds } }) .select('_id') .lean(); - return existing.map((doc) => doc._id.toString()); + const deploymentIds = getDeploymentSkillIds() + .map((id) => id.toString()) + .filter((id) => validIds.includes(id)); + return [...existing.map((doc) => doc._id.toString()), ...deploymentIds]; }, - findAccessibleSkillIds: () => - findAccessibleResources({ - userId: user.id, - role: user.role, - resourceType: ResourceType.SKILL, - requiredPermissions: PermissionBits.VIEW, - }), + findAccessibleSkillIds: async () => + mergeDeploymentSkillIds( + await findAccessibleResources({ + userId: user.id, + role: user.role, + resourceType: ResourceType.SKILL, + requiredPermissions: PermissionBits.VIEW, + }), + ), }; } diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index ae81afb1f2..8e65d6b10c 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -150,7 +150,9 @@ jest.mock('~/server/services/Files/permissions', () => ({ jest.mock('~/server/services/Endpoints/agents/skillDeps', () => ({ getSkillToolDeps: mockGetSkillToolDeps, + getSkillDbMethods: jest.fn(() => ({})), canAuthorSkillFiles: mockCanAuthorSkillFiles, + withDeploymentSkillIds: jest.fn((ids = []) => ids), enrichWithSkillConfigurable: mockEnrichWithSkillConfigurable, buildSkillPrimedIdsByName: mockBuildSkillPrimedIdsByName, buildAgentToolContext: mockBuildAgentToolContext, diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index 8c3a3dbaba..f89de34d4d 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -200,7 +200,9 @@ jest.mock('~/server/services/Files/permissions', () => ({ jest.mock('~/server/services/Endpoints/agents/skillDeps', () => ({ getSkillToolDeps: mockGetSkillToolDeps, + getSkillDbMethods: jest.fn(() => ({})), canAuthorSkillFiles: mockCanAuthorSkillFiles, + withDeploymentSkillIds: jest.fn((ids = []) => ids), enrichWithSkillConfigurable: mockEnrichWithSkillConfigurable, buildSkillPrimedIdsByName: mockBuildSkillPrimedIdsByName, buildAgentToolContext: mockBuildAgentToolContext, diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index be66412f3e..89422b2eaa 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -47,7 +47,9 @@ const { } = require('~/server/services/PermissionService'); const { getSkillToolDeps, + getSkillDbMethods, canAuthorSkillFiles, + withDeploymentSkillIds, buildAgentToolContext, enrichLoadedToolsWithAgentContext, } = require('~/server/services/Endpoints/agents/skillDeps'); @@ -229,6 +231,7 @@ const OpenAIChatCompletionController = async (req, res) => { endpoint: agent.provider, model_parameters: agent.model_parameters ?? {}, }; + const skillDbMethods = getSkillDbMethods(); // `filterFilesByAgentAccess` is intentionally omitted: it calls // `checkPermission` with `resourceType: AGENT`, but this route @@ -246,21 +249,23 @@ const OpenAIChatCompletionController = async (req, res) => { getUserCodeFiles: db.getUserCodeFiles, getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, - listSkillsByAccess: db.listSkillsByAccess, - listAlwaysApplySkills: db.listAlwaysApplySkills, - getSkillByName: db.getSkillByName, + listSkillsByAccess: skillDbMethods.listSkillsByAccess, + listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, + getSkillByName: skillDbMethods.getSkillByName, }; const enabledCapabilities = new Set(agentsEConfig?.capabilities); const skillsCapabilityEnabled = enabledCapabilities.has(AgentCapabilities.skills); const ephemeralSkillsToggle = req.body?.ephemeralAgent?.skills === true; const accessibleSkillIds = skillsCapabilityEnabled - ? await findAccessibleResources({ - userId: req.user.id, - role: req.user.role, - resourceType: ResourceType.SKILL, - requiredPermissions: PermissionBits.VIEW, - }) + ? withDeploymentSkillIds( + await findAccessibleResources({ + userId: req.user.id, + role: req.user.role, + resourceType: ResourceType.SKILL, + requiredPermissions: PermissionBits.VIEW, + }), + ) : []; const editableSkillIds = skillsCapabilityEnabled ? await findAccessibleResources({ diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 74972964da..9022c1e6f0 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -56,7 +56,9 @@ const { } = require('~/server/services/PermissionService'); const { getSkillToolDeps, + getSkillDbMethods, canAuthorSkillFiles, + withDeploymentSkillIds, buildAgentToolContext, enrichLoadedToolsWithAgentContext, } = require('~/server/services/Endpoints/agents/skillDeps'); @@ -351,6 +353,7 @@ const createResponse = async (req, res) => { // Create tool loader const loadTools = createToolLoader(abortController.signal); + const skillDbMethods = getSkillDbMethods(); // Initialize the agent first to check for disableStreaming const endpointOption = { @@ -374,9 +377,9 @@ const createResponse = async (req, res) => { getUserCodeFiles: db.getUserCodeFiles, getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, - listSkillsByAccess: db.listSkillsByAccess, - listAlwaysApplySkills: db.listAlwaysApplySkills, - getSkillByName: db.getSkillByName, + listSkillsByAccess: skillDbMethods.listSkillsByAccess, + listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, + getSkillByName: skillDbMethods.getSkillByName, }; const enabledCapabilities = new Set( @@ -385,12 +388,14 @@ const createResponse = async (req, res) => { const skillsCapabilityEnabled = enabledCapabilities.has(AgentCapabilities.skills); const ephemeralSkillsToggle = req.body?.ephemeralAgent?.skills === true; const accessibleSkillIds = skillsCapabilityEnabled - ? await findAccessibleResources({ - userId: req.user.id, - role: req.user.role, - resourceType: ResourceType.SKILL, - requiredPermissions: PermissionBits.VIEW, - }) + ? withDeploymentSkillIds( + await findAccessibleResources({ + userId: req.user.id, + role: req.user.role, + resourceType: ResourceType.SKILL, + requiredPermissions: PermissionBits.VIEW, + }), + ) : []; const editableSkillIds = skillsCapabilityEnabled ? await findAccessibleResources({ diff --git a/api/server/index.js b/api/server/index.js index 20fa0f9bf1..648d099143 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -21,6 +21,7 @@ const { GenerationJobManager, createStreamServices, initializeFileStorage, + initializeDeploymentSkills, preAuthTenantMiddleware, setupGracefulShutdown, updateInterfacePermissions, @@ -116,6 +117,7 @@ const startServer = async () => { }); const appConfig = await getAppConfig({ baseOnly: true }); initializeFileStorage(appConfig); + await initializeDeploymentSkills({ projectRoot: path.resolve(__dirname, '../..') }); startExpiredFileSweep({ appConfig, loadAppConfig: getAppConfig }); await runAsSystem(async () => { await performStartupChecks(appConfig); diff --git a/api/server/middleware/accessResources/canAccessSkillResource.js b/api/server/middleware/accessResources/canAccessSkillResource.js index 1010ca8998..d68612df27 100644 --- a/api/server/middleware/accessResources/canAccessSkillResource.js +++ b/api/server/middleware/accessResources/canAccessSkillResource.js @@ -1,6 +1,7 @@ -const { ResourceType } = require('librechat-data-provider'); +const { ResourceType, PermissionBits } = require('librechat-data-provider'); const { canAccessResource } = require('./canAccessResource'); const { getSkillById } = require('~/models'); +const { getDeploymentSkillById } = require('@librechat/api'); /** * Skill-specific middleware factory that checks skill access permissions. @@ -19,12 +20,35 @@ const canAccessSkillResource = (options) => { throw new Error('canAccessSkillResource: requiredPermission is required and must be a number'); } - return canAccessResource({ + const aclMiddleware = canAccessResource({ resourceType: ResourceType.SKILL, requiredPermission, resourceIdParam, idResolver: getSkillById, }); + + return (req, res, next) => { + const rawResourceId = req.params[resourceIdParam]; + const deploymentSkill = rawResourceId ? getDeploymentSkillById(rawResourceId) : null; + if (!deploymentSkill) { + return aclMiddleware(req, res, next); + } + if (requiredPermission !== PermissionBits.VIEW) { + return res.status(403).json({ + error: 'Forbidden', + message: 'Deployment skills are read-only', + }); + } + req.resourceAccess = { + resourceType: ResourceType.SKILL, + resourceId: deploymentSkill._id, + customResourceId: rawResourceId, + permission: requiredPermission, + userId: req.user?.id, + resourceInfo: deploymentSkill, + }; + return next(); + }; }; module.exports = { diff --git a/api/server/routes/skills.js b/api/server/routes/skills.js index 694009a480..eb756a432f 100644 --- a/api/server/routes/skills.js +++ b/api/server/routes/skills.js @@ -21,14 +21,11 @@ const { const { createSkill, getSkillById, - listSkillsByAccess, updateSkill, deleteSkill, - listSkillFiles, upsertSkillFile, deleteSkillFile, getSkillFileByPath, - updateSkillFileContent, getRoleByName, } = require('~/models'); const { requireJwtAuth, canAccessSkillResource } = require('~/server/middleware'); @@ -42,6 +39,11 @@ const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { createFileLimiters } = require('~/server/middleware/limiters/uploadLimiters'); const configMiddleware = require('~/server/middleware/config/app'); const { getFileStrategy } = require('~/server/utils/getFileStrategy'); +const { + getSkillDbMethods, + withDeploymentSkillIds, + getSkillStrategyFunctions, +} = require('~/server/services/Endpoints/agents/skillDeps'); const router = express.Router(); @@ -100,6 +102,7 @@ const checkSkillCreate = generateCheckAccess({ // Rate limiters (reuse existing file upload limiters) // --------------------------------------------------------------------------- const { fileUploadIpLimiter, fileUploadUserLimiter } = createFileLimiters(); +const skillDbMethods = getSkillDbMethods(); router.use(requireJwtAuth); router.use(configMiddleware); @@ -110,18 +113,28 @@ router.use(checkSkillAccess); // --------------------------------------------------------------------------- const handlers = createSkillsHandlers({ createSkill, - getSkillById, - listSkillsByAccess, + getSkillById: skillDbMethods.getSkillById, + listSkillsByAccess: skillDbMethods.listSkillsByAccess, updateSkill, deleteSkill, - listSkillFiles, + listSkillFiles: skillDbMethods.listSkillFiles, deleteSkillFile, - getSkillFileByPath, - updateSkillFileContent, - getStrategyFunctions, - findAccessibleResources, - findPubliclyAccessibleResources, - hasPublicPermission, + getSkillFileByPath: skillDbMethods.getSkillFileByPath, + updateSkillFileContent: skillDbMethods.updateSkillFileContent, + getStrategyFunctions: getSkillStrategyFunctions, + findAccessibleResources: async (params) => + params.resourceType === 'skill' && params.requiredPermissions === PermissionBits.VIEW + ? withDeploymentSkillIds(await findAccessibleResources(params)) + : findAccessibleResources(params), + findPubliclyAccessibleResources: async (params) => + params.resourceType === 'skill' && params.requiredPermissions === PermissionBits.VIEW + ? withDeploymentSkillIds(await findPubliclyAccessibleResources(params)) + : findPubliclyAccessibleResources(params), + hasPublicPermission: async (params) => + params.resourceType === 'skill' && params.requiredPermissions === PermissionBits.VIEW + ? withDeploymentSkillIds([]).some((id) => id.toString() === params.resourceId.toString()) || + hasPublicPermission(params) + : hasPublicPermission(params), grantPermission, isValidObjectIdString, }); diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 25b00d05a2..225634f01d 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -32,7 +32,9 @@ const { loadAgentTools, loadToolsForExecution } = require('~/server/services/Too const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); const { getSkillToolDeps, + getSkillDbMethods, canAuthorSkillFiles, + withDeploymentSkillIds, buildAgentToolContext, enrichLoadedToolsWithAgentContext, } = require('./skillDeps'); @@ -143,14 +145,17 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { const skillsCapabilityEnabled = enabledCapabilities.has(AgentCapabilities.skills); const codeEnvAvailable = enabledCapabilities.has(AgentCapabilities.execute_code); const ephemeralSkillsToggle = req.body?.ephemeralAgent?.skills === true; + const skillDbMethods = getSkillDbMethods(); const accessibleSkillIds = skillsCapabilityEnabled - ? await findAccessibleResources({ - userId: req.user.id, - role: req.user.role, - resourceType: ResourceType.SKILL, - requiredPermissions: PermissionBits.VIEW, - }) + ? withDeploymentSkillIds( + await findAccessibleResources({ + userId: req.user.id, + role: req.user.role, + resourceType: ResourceType.SKILL, + requiredPermissions: PermissionBits.VIEW, + }), + ) : []; const editableSkillIds = skillsCapabilityEnabled ? await findAccessibleResources({ @@ -368,9 +373,9 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, - listSkillsByAccess: db.listSkillsByAccess, - listAlwaysApplySkills: db.listAlwaysApplySkills, - getSkillByName: db.getSkillByName, + listSkillsByAccess: skillDbMethods.listSkillsByAccess, + listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, + getSkillByName: skillDbMethods.getSkillByName, }, ); @@ -439,9 +444,9 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, - listSkillsByAccess: db.listSkillsByAccess, - listAlwaysApplySkills: db.listAlwaysApplySkills, - getSkillByName: db.getSkillByName, + listSkillsByAccess: skillDbMethods.listSkillsByAccess, + listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, + getSkillByName: skillDbMethods.getSkillByName, }, // The callback fires during BFS, before the helper prunes agents // whose edges end up filtered. Don't populate `agentConfigs` here — @@ -646,9 +651,9 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, - listSkillsByAccess: db.listSkillsByAccess, - listAlwaysApplySkills: db.listAlwaysApplySkills, - getSkillByName: db.getSkillByName, + listSkillsByAccess: skillDbMethods.listSkillsByAccess, + listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills, + getSkillByName: skillDbMethods.getSkillByName, }, ); agentConfigs.set(agentId, config); diff --git a/api/server/services/Endpoints/agents/skillDeps.js b/api/server/services/Endpoints/agents/skillDeps.js index 27b8eeff66..6a8e64361b 100644 --- a/api/server/services/Endpoints/agents/skillDeps.js +++ b/api/server/services/Endpoints/agents/skillDeps.js @@ -12,6 +12,10 @@ const { getStorageMetadata, resolveRequestTenantId, enrichWithSkillConfigurable, + mergeDeploymentSkillIds, + createDeploymentSkillMethods, + isDeploymentSkillFileSource, + getDeploymentSkillDownloadStream, } = require('@librechat/api'); const { Permissions, @@ -27,6 +31,34 @@ const { checkPermission, grantPermission } = require('~/server/services/Permissi const { getFileStrategy } = require('~/server/utils/getFileStrategy'); const db = require('~/models'); +const deploymentSkillMethods = createDeploymentSkillMethods({ + getSkillById: db.getSkillById, + getSkillByName: db.getSkillByName, + listSkillsByAccess: db.listSkillsByAccess, + listAlwaysApplySkills: db.listAlwaysApplySkills, + listSkillFiles: db.listSkillFiles, + getSkillFileByPath: db.getSkillFileByPath, + updateSkillFileContent: db.updateSkillFileContent, + updateSkillFileCodeEnvIds: db.updateSkillFileCodeEnvIds, +}); + +function getSkillDbMethods() { + return deploymentSkillMethods; +} + +function withDeploymentSkillIds(ids = []) { + return mergeDeploymentSkillIds(ids); +} + +function getSkillStrategyFunctions(source) { + if (isDeploymentSkillFileSource(source)) { + return { + getDownloadStream: (_req, filepath) => getDeploymentSkillDownloadStream(filepath), + }; + } + return getStrategyFunctions(source); +} + function resolveSkillStorage(req, { isImage = false } = {}) { const source = getFileStrategy(req.config, { context: FileContext.skill_file, isImage }); const strategy = getStrategyFunctions(source); @@ -290,7 +322,7 @@ function enrichLoadedToolsWithAgentContext({ result, req, ctx = {}, fallback = { /** Skill-related properties for ToolExecuteOptions (stable references, allocated once). */ const skillToolDeps = { - getSkillByName: db.getSkillByName, + getSkillByName: deploymentSkillMethods.getSkillByName, getAuthorSkillByName, createSkill: db.createSkill, updateSkill: db.updateSkill, @@ -299,14 +331,14 @@ const skillToolDeps = { canEditSkill, grantSkillOwner, saveSkillFileContent, - listSkillFiles: db.listSkillFiles, - getStrategyFunctions, + listSkillFiles: deploymentSkillMethods.listSkillFiles, + getStrategyFunctions: getSkillStrategyFunctions, batchUploadCodeEnvFiles, getSessionInfo, checkIfActive, - updateSkillFileCodeEnvIds: db.updateSkillFileCodeEnvIds, - getSkillFileByPath: db.getSkillFileByPath, - updateSkillFileContent: db.updateSkillFileContent, + updateSkillFileCodeEnvIds: deploymentSkillMethods.updateSkillFileCodeEnvIds, + getSkillFileByPath: deploymentSkillMethods.getSkillFileByPath, + updateSkillFileContent: deploymentSkillMethods.updateSkillFileContent, /** * `read_file` falls back to a sandbox `cat` for `/mnt/data/...` paths * and for `{firstSegment}/...` paths whose first segment isn't a known @@ -327,6 +359,9 @@ module.exports = { getSkillToolDeps, canAuthorSkillFiles, isAgentSkillsEnabledForRun, + getSkillDbMethods, + withDeploymentSkillIds, + getSkillStrategyFunctions, enrichWithSkillConfigurable, buildSkillPrimedIdsByName, buildAgentToolContext, diff --git a/api/server/services/Endpoints/agents/skillDeps.spec.js b/api/server/services/Endpoints/agents/skillDeps.spec.js index 2f221f89e6..3782a4664a 100644 --- a/api/server/services/Endpoints/agents/skillDeps.spec.js +++ b/api/server/services/Endpoints/agents/skillDeps.spec.js @@ -4,6 +4,7 @@ const mockGetStrategyFunctions = jest.fn(); const mockGetFileStrategy = jest.fn(); const mockGetStorageMetadata = jest.fn(); const mockResolveRequestTenantId = jest.fn(); +const mockCreateDeploymentSkillMethods = jest.fn((methods) => methods); jest.mock('~/server/services/Files/strategies', () => ({ getStrategyFunctions: (...args) => mockGetStrategyFunctions(...args), @@ -22,8 +23,12 @@ jest.mock('~/server/services/Files/Code/process', () => ({ jest.mock('@librechat/api', () => ({ checkAccess: jest.fn(), + createDeploymentSkillMethods: (...args) => mockCreateDeploymentSkillMethods(...args), enrichWithSkillConfigurable: jest.fn(), + getDeploymentSkillDownloadStream: jest.fn(), getStorageMetadata: (...args) => mockGetStorageMetadata(...args), + isDeploymentSkillFileSource: jest.fn(() => false), + mergeDeploymentSkillIds: jest.fn((ids = []) => ids), resolveRequestTenantId: (...args) => mockResolveRequestTenantId(...args), })); diff --git a/deploy-compose.yml b/deploy-compose.yml index 5d7bcebcd0..72ce653f9a 100644 --- a/deploy-compose.yml +++ b/deploy-compose.yml @@ -30,6 +30,7 @@ services: - ./images:/app/client/public/images - ./uploads:/app/uploads - ./logs:/app/api/logs + - ./skill:/app/skill client: image: nginx:1.27.0-alpine diff --git a/docker-compose.yml b/docker-compose.yml index 6c95c309b4..dde7b653d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - ./images:/app/client/public/images - ./uploads:/app/uploads - ./logs:/app/logs + - ./skill:/app/skill mongodb: container_name: chat-mongodb image: mongo:8.0.20 diff --git a/e2e/fixtures/deployment-skills/e2e-deployment-skill/SKILL.md b/e2e/fixtures/deployment-skills/e2e-deployment-skill/SKILL.md new file mode 100644 index 0000000000..a0d1d260ec --- /dev/null +++ b/e2e/fixtures/deployment-skills/e2e-deployment-skill/SKILL.md @@ -0,0 +1,10 @@ +--- +name: e2e-deployment-skill +description: Use this deployment skill to verify shared skills load during Playwright startup. +always-apply: true +user-invocable: true +--- + +# E2E Deployment Skill + +E2E deployment skill loaded through Playwright from the configured deployment skills directory. diff --git a/e2e/fixtures/deployment-skills/e2e-deployment-skill/guide.txt b/e2e/fixtures/deployment-skills/e2e-deployment-skill/guide.txt new file mode 100644 index 0000000000..17ca54fd45 --- /dev/null +++ b/e2e/fixtures/deployment-skills/e2e-deployment-skill/guide.txt @@ -0,0 +1 @@ +deployment skill file fixture diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts index 08284efeed..1ef0b22a73 100644 --- a/e2e/playwright.config.mock.ts +++ b/e2e/playwright.config.mock.ts @@ -9,6 +9,7 @@ const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js'); const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml'); const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml'); const reportPath = path.resolve(rootPath, 'e2e/playwright-report'); +const deploymentSkillsPath = path.resolve(rootPath, 'e2e/fixtures/deployment-skills'); const baseURL = getE2EBaseURL(); const chromiumChannel = process.env.E2E_CHROMIUM_CHANNEL || undefined; @@ -27,6 +28,7 @@ const vanillaOverrides = { const baseEnv = { ...getLocalE2EEnv(), CONFIG_PATH: configPath, + DEPLOYMENT_SKILLS_DIR: deploymentSkillsPath, /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */ LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath, ...vanillaOverrides, diff --git a/e2e/specs/mock/deployment-skills.spec.ts b/e2e/specs/mock/deployment-skills.spec.ts new file mode 100644 index 0000000000..c5c98e2926 --- /dev/null +++ b/e2e/specs/mock/deployment-skills.spec.ts @@ -0,0 +1,218 @@ +import { expect, test } from '@playwright/test'; +import type { Page } from '@playwright/test'; +import { NEW_CHAT_PATH } from './helpers'; + +const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill'; +const DEPLOYMENT_SKILL_DESCRIPTION = + 'Use this deployment skill to verify shared skills load during Playwright startup.'; + +type RefreshTokenBody = { + token?: string; +}; + +type SkillSummary = { + _id: string; + name: string; + description: string; + source?: string; + sourceMetadata?: Record; + fileCount?: number; + alwaysApply?: boolean; + isPublic?: boolean; +}; + +type SkillDetail = SkillSummary & { + body: string; + frontmatter?: Record; +}; + +type SkillFile = { + _id: string; + skillId: string; + relativePath: string; + file_id: string; + filename: string; + filepath: string; + source: string; + mimeType: string; + bytes: number; + category: string; + isExecutable: boolean; + author: string; + createdAt: string; + updatedAt: string; + content?: string; +}; + +type SkillFileContent = Pick< + SkillFile, + 'relativePath' | 'filename' | 'mimeType' | 'bytes' | 'content' +> & { + isBinary?: boolean; +}; + +type ApiResult = { + ok: boolean; + status: number; + text: string; + json: T | null; +}; + +async function getAccessToken(page: Page): Promise { + const result = await page.evaluate(async () => { + const response = await fetch('/api/auth/refresh', { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + const text = await response.text(); + let json: unknown = null; + try { + json = text ? JSON.parse(text) : null; + } catch { + json = null; + } + return { ok: response.ok, status: response.status, text, json }; + }); + + if (!result.ok) { + throw new Error( + `Expected /api/auth/refresh to return 2xx, got ${result.status}: ${result.text}`, + ); + } + + const body = result.json as RefreshTokenBody | null; + if (!body?.token) { + throw new Error(`Expected /api/auth/refresh to return a token, got: ${result.text}`); + } + + return body.token; +} + +async function apiJson( + page: Page, + path: string, + token: string, + init: { method?: string; body?: unknown } = {}, +): Promise> { + return page.evaluate( + async ({ accessToken, body, method, urlPath }) => { + const headers: Record = { Authorization: `Bearer ${accessToken}` }; + if (body !== undefined) { + headers['Content-Type'] = 'application/json'; + } + const response = await fetch(urlPath, { + method, + credentials: 'include', + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const text = await response.text(); + let json: unknown = null; + try { + json = text ? JSON.parse(text) : null; + } catch { + json = null; + } + return { ok: response.ok, status: response.status, text, json }; + }, + { + accessToken: token, + body: init.body, + method: init.method, + urlPath: path, + }, + ) as Promise>; +} + +async function fetchJson(page: Page, path: string, token: string): Promise { + const result = await apiJson(page, path, token); + if (!result.ok) { + throw new Error(`Expected ${path} to return 2xx, got ${result.status}: ${result.text}`); + } + return result.json as T; +} + +test.describe('deployment skills', () => { + test('loads configured deployment skills for every authenticated user as read-only', async ({ + page, + }) => { + await page.goto(NEW_CHAT_PATH, { timeout: 10000 }); + const token = await getAccessToken(page); + + const list = await fetchJson<{ skills?: SkillSummary[] }>( + page, + `/api/skills?search=${encodeURIComponent(DEPLOYMENT_SKILL_NAME)}&limit=10`, + token, + ); + const skill = list.skills?.find((item) => item.name === DEPLOYMENT_SKILL_NAME); + expect(skill).toMatchObject({ + name: DEPLOYMENT_SKILL_NAME, + description: DEPLOYMENT_SKILL_DESCRIPTION, + source: 'deployment', + sourceMetadata: { deployment: true }, + fileCount: 1, + alwaysApply: true, + isPublic: true, + }); + + const detail = await fetchJson( + page, + `/api/skills/${encodeURIComponent(skill!._id)}`, + token, + ); + expect(detail.body).toContain('E2E deployment skill loaded through Playwright'); + expect(detail.frontmatter).toMatchObject({ + name: DEPLOYMENT_SKILL_NAME, + description: DEPLOYMENT_SKILL_DESCRIPTION, + 'always-apply': true, + }); + + const files = await fetchJson<{ files?: SkillFile[] }>( + page, + `/api/skills/${encodeURIComponent(skill!._id)}/files`, + token, + ); + expect(files.files).toHaveLength(1); + expect(files.files?.[0]).toMatchObject({ + skillId: skill!._id, + relativePath: 'guide.txt', + filename: 'guide.txt', + source: 'deployment', + mimeType: 'text/plain', + bytes: 'deployment skill file fixture\n'.length, + category: 'other', + isExecutable: false, + }); + expect(files.files?.[0]).not.toHaveProperty('content'); + + const downloaded = await fetchJson( + page, + `/api/skills/${encodeURIComponent(skill!._id)}/files/guide.txt`, + token, + ); + expect(downloaded).toMatchObject({ + relativePath: 'guide.txt', + filename: 'guide.txt', + mimeType: 'text/plain', + bytes: 'deployment skill file fixture\n'.length, + isBinary: false, + content: 'deployment skill file fixture\n', + }); + + const patch = await apiJson<{ message?: string }>( + page, + `/api/skills/${encodeURIComponent(skill!._id)}`, + token, + { + method: 'PATCH', + body: { + description: 'Deployment skills should stay read-only.', + }, + }, + ); + expect(patch.status).toBe(403); + expect(patch.json).toMatchObject({ message: 'Deployment skills are read-only' }); + }); +}); diff --git a/packages/api/src/agents/__tests__/skills.test.ts b/packages/api/src/agents/__tests__/skills.test.ts index b09b39685b..df21c665b1 100644 --- a/packages/api/src/agents/__tests__/skills.test.ts +++ b/packages/api/src/agents/__tests__/skills.test.ts @@ -651,6 +651,30 @@ describe('resolveSkillActive', () => { ).toBe(false); }); + it('respects explicit override = false even for deployment skills', () => { + const deploymentSkill = { ...makeSkill(new Types.ObjectId()), deployment: true }; + expect( + resolveSkillActive({ + skill: deploymentSkill, + skillStates: { [deploymentSkill._id.toString()]: false }, + userId: undefined, + defaultActiveOnShare: true, + }), + ).toBe(false); + }); + + it('defaults deployment skills to active without ownership or shared defaults', () => { + const deploymentSkill = { ...makeSkill(new Types.ObjectId()), deployment: true }; + expect( + resolveSkillActive({ + skill: deploymentSkill, + skillStates: {}, + userId: undefined, + defaultActiveOnShare: false, + }), + ).toBe(true); + }); + it('owned skills default to active when no override is present', () => { const userObjectId = new Types.ObjectId(); const userId = userObjectId.toString(); @@ -1138,6 +1162,7 @@ describe('resolveManualSkills', () => { author: Types.ObjectId; allowedTools?: string[]; userInvocable?: boolean; + deployment?: boolean; }; const buildGetSkillByName = @@ -1347,6 +1372,21 @@ describe('resolveManualSkills', () => { expect(result).toEqual([{ _id: shared._id, name: 'shared', body: 'shared-body' }]); }); + it('allows deployment skills even when shared skills default inactive', async () => { + const deployment = { + ...mkSkill('deployment', otherAuthor, 'deployment-body'), + deployment: true, + }; + const result = await resolveManualSkills({ + names: ['deployment'], + getSkillByName: buildGetSkillByName({ deployment }), + accessibleSkillIds: [deployment._id], + userId, + defaultActiveOnShare: false, + }); + expect(result).toEqual([{ _id: deployment._id, name: 'deployment', body: 'deployment-body' }]); + }); + it('drops explicitly-deactivated skills (skillStates override wins over ownership default)', async () => { const owned = mkSkill('owned-off', userOid); const result = await resolveManualSkills({ @@ -1781,6 +1821,7 @@ describe('resolveAlwaysApplySkills', () => { body: string; author: Types.ObjectId | string; allowedTools?: string[]; + deployment?: boolean; }; const mkRow = ( @@ -1868,6 +1909,22 @@ describe('resolveAlwaysApplySkills', () => { expect(result).toEqual([{ _id: shared._id, name: 'shared-on', body: 'shared-body' }]); }); + it('allows deployment always-apply skills even when shared skills default inactive', async () => { + const deployment: AlwaysApplyRow = { + ...mkRow('deployment-always', otherAuthor, 'deployment body'), + deployment: true, + }; + const result = await resolveAlwaysApplySkills({ + listAlwaysApplySkills: buildLister([deployment]), + accessibleSkillIds: [deployment._id], + userId, + defaultActiveOnShare: false, + }); + expect(result).toEqual([ + { _id: deployment._id, name: 'deployment-always', body: 'deployment body' }, + ]); + }); + it('honors explicit deactivation override even for owned skills', async () => { const owned = mkRow('owned-off', userOid); const result = await resolveAlwaysApplySkills({ diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index 4306ee1601..1c9787d8de 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -82,6 +82,8 @@ export interface ToolExecuteOptions { * prior cache entry. */ version: number; fileCount: number; + /** True for deployment-directory skills that are loaded in memory. */ + deployment?: boolean; /** * Set when the skill author opted out of model invocation. The handler * rejects the call and returns an instructive error so the model knows diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index ddcc528d12..36cdb5e53f 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -367,6 +367,8 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods { * by the manual-invocation resolver. Defaults to `true`. */ userInvocable?: boolean; + /** True for deployment-directory skills that are loaded in memory. */ + deployment?: boolean; }>; has_more?: boolean; after?: string | null; @@ -412,6 +414,8 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods { * caller can't bypass the popover-side filter. */ userInvocable?: boolean; + /** True for deployment-directory skills that are loaded in memory. */ + deployment?: boolean; } | null>; /** * Load accessible skills with `alwaysApply: true`, eagerly including @@ -431,6 +435,8 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods { body: string; author: import('mongoose').Types.ObjectId; allowedTools?: string[]; + /** True for deployment-directory skills that are loaded in memory. */ + deployment?: boolean; }>; has_more?: boolean; after?: string | null; diff --git a/packages/api/src/agents/skills.ts b/packages/api/src/agents/skills.ts index 188bcf4af5..102ea14dc3 100644 --- a/packages/api/src/agents/skills.ts +++ b/packages/api/src/agents/skills.ts @@ -274,7 +274,7 @@ export function resolveAgentScopedSkillIds( export interface ResolveSkillActiveParams { /** Skill being evaluated. Only `_id` and `author` matter for resolution. */ - skill: { _id: Types.ObjectId | string; author: Types.ObjectId | string }; + skill: { _id: Types.ObjectId | string; author: Types.ObjectId | string; deployment?: boolean }; /** Per-user overrides: `{ [skillId]: boolean }`. Missing entries use the default. */ skillStates?: Record; /** Current user ID. When absent, the function fails closed for all non-overridden skills. */ @@ -299,6 +299,9 @@ export function resolveSkillActive(params: ResolveSkillActiveParams): boolean { if (override !== undefined) { return override; } + if (skill.deployment === true) { + return true; + } if (!userId) { return false; } @@ -615,6 +618,7 @@ export interface ResolveManualSkillsParams { name: string; body: string; author: Types.ObjectId | string; + deployment?: boolean; /** * Skill-declared tool allowlist, forwarded verbatim from the skill doc. * Surfaced on `ResolvedManualSkill` so future runtime enforcement can @@ -785,7 +789,7 @@ export async function resolveManualSkills( return null; } const active = resolveSkillActive({ - skill: { _id: skill._id, author: skill.author }, + skill: { _id: skill._id, author: skill.author, deployment: skill.deployment }, skillStates, userId, defaultActiveOnShare, @@ -835,6 +839,7 @@ export interface ResolveAlwaysApplySkillsParams { body: string; author: Types.ObjectId | string; allowedTools?: string[]; + deployment?: boolean; }>; has_more?: boolean; after?: string | null; @@ -933,7 +938,7 @@ export async function resolveAlwaysApplySkills( continue; } const active = resolveSkillActive({ - skill: { _id: skill._id, author: skill.author }, + skill: { _id: skill._id, author: skill.author, deployment: skill.deployment }, skillStates, userId, defaultActiveOnShare, diff --git a/packages/api/src/skills/__tests__/deployment.test.ts b/packages/api/src/skills/__tests__/deployment.test.ts new file mode 100644 index 0000000000..5843bd202d --- /dev/null +++ b/packages/api/src/skills/__tests__/deployment.test.ts @@ -0,0 +1,327 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { Types } from 'mongoose'; +import type { CodeEnvRef } from 'librechat-data-provider'; +import { + DEPLOYMENT_SKILLS_DIR_ENV, + createDeploymentSkillMethods, + getDeploymentSkillIds, + initializeDeploymentSkills, + loadDeploymentSkillsFromDirectory, + mergeDeploymentSkillIds, + resolveDeploymentSkillDirectory, +} from '../deployment'; +import type { DeploymentSkillBaseMethods } from '../deployment'; + +const DESCRIPTION = 'Use this skill when the deployment needs a shared testing fixture.'; + +let tempRoots: string[] = []; + +async function makeTempRoot(): Promise { + const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'deployment-skills-')); + tempRoots.push(root); + return root; +} + +async function writeDeploymentSkill( + root: string, + options: { + skillsDir?: string; + folder?: string; + name?: string; + alwaysApply?: boolean; + frontmatter?: string; + } = {}, +): Promise { + const skillsDir = options.skillsDir ?? 'skill'; + const folder = options.folder ?? options.name ?? 'analysis-kit'; + const name = options.name ?? folder; + const skillDir = path.join(root, skillsDir, folder); + await fs.promises.mkdir(path.join(skillDir, 'references'), { recursive: true }); + await fs.promises.mkdir(path.join(skillDir, 'assets'), { recursive: true }); + await fs.promises.writeFile( + path.join(skillDir, 'SKILL.md'), + options.frontmatter ?? + [ + '---', + `name: ${name}`, + `description: ${DESCRIPTION}`, + 'allowed-tools:', + ' - execute_code', + 'disable-model-invocation: true', + 'user-invocable: false', + `always-apply: ${options.alwaysApply === true ? 'true' : 'false'}`, + '---', + '', + '# Analysis Kit', + '', + 'Read references/guide.txt before answering.', + ].join('\n'), + ); + await fs.promises.writeFile(path.join(skillDir, 'references', 'guide.txt'), 'reference notes'); + await fs.promises.writeFile(path.join(skillDir, 'assets', 'pixel.bin'), Buffer.from([0, 1, 2])); + return skillDir; +} + +afterEach(async () => { + const emptyRoot = await makeTempRoot(); + await initializeDeploymentSkills({ projectRoot: emptyRoot, env: {} }); + await Promise.all( + tempRoots.map((root) => fs.promises.rm(root, { recursive: true, force: true })), + ); + tempRoots = []; +}); + +describe('resolveDeploymentSkillDirectory', () => { + it('defaults to project root ./skill', () => { + const root = path.join(os.tmpdir(), 'librechat-root'); + expect(resolveDeploymentSkillDirectory({ projectRoot: root, env: {} })).toEqual({ + directory: path.join(root, 'skill'), + explicitlyConfigured: false, + }); + }); + + it('honors a relative DEPLOYMENT_SKILLS_DIR override', () => { + const root = path.join(os.tmpdir(), 'librechat-root'); + const env = { [DEPLOYMENT_SKILLS_DIR_ENV]: 'config/skills' }; + expect(resolveDeploymentSkillDirectory({ projectRoot: root, env })).toEqual({ + directory: path.join(root, 'config', 'skills'), + explicitlyConfigured: true, + }); + }); +}); + +describe('loadDeploymentSkillsFromDirectory', () => { + it('treats a missing default directory as an empty deployment catalog', async () => { + const root = await makeTempRoot(); + const registry = await loadDeploymentSkillsFromDirectory(path.join(root, 'skill'), { + projectRoot: root, + }); + expect(registry.list()).toEqual([]); + }); + + it('fails startup validation when an explicitly configured directory is missing', async () => { + const root = await makeTempRoot(); + await expect( + loadDeploymentSkillsFromDirectory(path.join(root, 'missing-skills'), { + projectRoot: root, + explicitlyConfigured: true, + }), + ).rejects.toThrow(/Deployment skills directory not found/); + }); + + it('loads bundled skills and their files without touching the DB', async () => { + const root = await makeTempRoot(); + await writeDeploymentSkill(root, { name: 'analysis-kit', alwaysApply: true }); + + const registry = await loadDeploymentSkillsFromDirectory(path.join(root, 'skill'), { + projectRoot: root, + }); + const [skill] = registry.list(); + + expect(skill).toMatchObject({ + name: 'analysis-kit', + description: DESCRIPTION, + source: 'deployment', + deployment: true, + isPublic: true, + alwaysApply: true, + allowedTools: ['execute_code'], + disableModelInvocation: true, + userInvocable: false, + authorName: 'Deployment', + sourceMetadata: { deployment: true, directory: 'skill/analysis-kit' }, + }); + expect(skill.fileCount).toBe(2); + expect(skill.files.map((file) => [file.relativePath, file.category])).toEqual([ + ['assets/pixel.bin', 'asset'], + ['references/guide.txt', 'reference'], + ]); + expect(skill.files.find((file) => file.relativePath === 'references/guide.txt')).toMatchObject({ + source: 'deployment', + content: 'reference notes', + isBinary: false, + mimeType: 'text/plain', + }); + expect(skill.files.find((file) => file.relativePath === 'assets/pixel.bin')).toMatchObject({ + isBinary: true, + }); + }); + + it('validates SKILL.md frontmatter at startup', async () => { + const root = await makeTempRoot(); + await writeDeploymentSkill(root, { + name: 'bad-frontmatter', + frontmatter: [ + '---', + 'name: bad-frontmatter', + `description: ${DESCRIPTION}`, + 'unknown-key: nope', + '---', + '', + 'Body', + ].join('\n'), + }); + + await expect( + loadDeploymentSkillsFromDirectory(path.join(root, 'skill'), { projectRoot: root }), + ).rejects.toThrow(/frontmatter\.unknown-key/); + }); + + it('validates bundled file paths at startup', async () => { + const root = await makeTempRoot(); + const skillDir = await writeDeploymentSkill(root, { name: 'bad-path' }); + await fs.promises.writeFile(path.join(skillDir, 'references', 'bad path.txt'), 'bad'); + + await expect( + loadDeploymentSkillsFromDirectory(path.join(root, 'skill'), { projectRoot: root }), + ).rejects.toThrow(/relativePath: Relative path contains invalid characters/); + }); + + it('rejects duplicate deployment skill names', async () => { + const root = await makeTempRoot(); + await writeDeploymentSkill(root, { folder: 'one', name: 'duplicate-name' }); + await writeDeploymentSkill(root, { folder: 'two', name: 'duplicate-name' }); + + await expect( + loadDeploymentSkillsFromDirectory(path.join(root, 'skill'), { projectRoot: root }), + ).rejects.toThrow(/Duplicate deployment skill name "duplicate-name"/); + }); +}); + +describe('createDeploymentSkillMethods', () => { + it('merges deployment skills into read paths while stripping them from DB calls', async () => { + const root = await makeTempRoot(); + await writeDeploymentSkill(root, { + skillsDir: 'config/skills', + name: 'analysis-kit', + alwaysApply: true, + }); + await initializeDeploymentSkills({ + projectRoot: root, + env: { [DEPLOYMENT_SKILLS_DIR_ENV]: 'config/skills' }, + }); + + const deploymentId = getDeploymentSkillIds()[0]; + const dbId = new Types.ObjectId(); + const dbAuthor = new Types.ObjectId(); + const dbSkill = { + _id: dbId, + name: 'db-skill', + description: 'A persisted skill.', + body: 'persisted body', + author: dbAuthor, + version: 3, + fileCount: 0, + updatedAt: new Date(0), + }; + const base: DeploymentSkillBaseMethods = { + getSkillById: jest.fn(async (id) => (id.toString() === dbId.toString() ? dbSkill : null)), + getSkillByName: jest.fn(async (name) => (name === dbSkill.name ? dbSkill : null)), + listSkillsByAccess: jest.fn(async () => ({ + skills: [dbSkill], + has_more: false, + after: null, + })), + listAlwaysApplySkills: jest.fn(async () => ({ + skills: [{ _id: dbId, name: 'db-always', body: 'db always', author: dbAuthor }], + has_more: false, + after: null, + })), + listSkillFiles: jest.fn(async () => []), + getSkillFileByPath: jest.fn(async () => null), + updateSkillFileContent: jest.fn(async () => undefined), + updateSkillFileCodeEnvIds: jest.fn(async (updates) => ({ + matchedCount: updates.length, + modifiedCount: updates.length, + })), + }; + + const methods = createDeploymentSkillMethods(base); + const mergedIds = mergeDeploymentSkillIds([dbId]); + + expect(mergedIds.map((id) => id.toString())).toEqual([ + dbId.toString(), + deploymentId.toString(), + ]); + + const deploymentSkill = await methods.getSkillById(deploymentId); + expect(deploymentSkill).toMatchObject({ name: 'analysis-kit', source: 'deployment' }); + expect(base.getSkillById).not.toHaveBeenCalled(); + + const listed = await methods.listSkillsByAccess?.({ + accessibleIds: mergedIds, + limit: 10, + }); + expect(listed?.skills.map((skill) => skill.name).sort()).toEqual(['analysis-kit', 'db-skill']); + expect(base.listSkillsByAccess).toHaveBeenCalledWith({ + accessibleIds: [dbId], + limit: 10, + }); + + const alwaysApply = await methods.listAlwaysApplySkills?.({ + accessibleIds: mergedIds, + limit: 10, + }); + expect(alwaysApply?.skills.map((skill) => skill.name).sort()).toEqual([ + 'analysis-kit', + 'db-always', + ]); + expect(base.listAlwaysApplySkills).toHaveBeenCalledWith({ + accessibleIds: [dbId], + limit: 10, + }); + + const files = await methods.listSkillFiles?.(deploymentId); + expect(files?.map((file) => file.relativePath).sort()).toEqual([ + 'assets/pixel.bin', + 'references/guide.txt', + ]); + const guideFile = files?.find((file) => file.relativePath === 'references/guide.txt'); + expect(guideFile?._id).toBeInstanceOf(Types.ObjectId); + expect(guideFile?.skillId.toString()).toBe(deploymentId.toString()); + expect(guideFile).toMatchObject({ + file_id: expect.any(String), + filename: 'guide.txt', + source: 'deployment', + mimeType: 'text/plain', + bytes: 'reference notes'.length, + category: 'reference', + isExecutable: false, + author: expect.any(Types.ObjectId), + }); + expect(guideFile?.createdAt.getTime()).toEqual(expect.any(Number)); + expect(guideFile?.updatedAt.getTime()).toEqual(expect.any(Number)); + expect(guideFile).not.toHaveProperty('content'); + expect(base.listSkillFiles).not.toHaveBeenCalled(); + + await methods.updateSkillFileContent?.(deploymentId, 'references/guide.txt', { + content: 'updated content', + isBinary: false, + }); + const updatedFile = await methods.getSkillFileByPath?.(deploymentId, 'references/guide.txt'); + expect(updatedFile?.content).toBe('updated content'); + expect(base.updateSkillFileContent).not.toHaveBeenCalled(); + + const codeEnvRef: CodeEnvRef = { + kind: 'skill', + id: deploymentId.toString(), + version: 1, + storage_session_id: 'storage-session', + file_id: 'file-id', + }; + const updateResult = await methods.updateSkillFileCodeEnvIds?.([ + { skillId: deploymentId, relativePath: 'references/guide.txt', codeEnvRef }, + { skillId: dbId, relativePath: 'references/db.txt', codeEnvRef }, + ]); + + expect(updateResult).toEqual({ matchedCount: 1, modifiedCount: 1 }); + expect(base.updateSkillFileCodeEnvIds).toHaveBeenCalledWith([ + { skillId: dbId, relativePath: 'references/db.txt', codeEnvRef }, + ]); + expect( + (await methods.getSkillFileByPath?.(deploymentId, 'references/guide.txt'))?.codeEnvRef, + ).toEqual(codeEnvRef); + }); +}); diff --git a/packages/api/src/skills/deployment.ts b/packages/api/src/skills/deployment.ts new file mode 100644 index 0000000000..d46cc69f42 --- /dev/null +++ b/packages/api/src/skills/deployment.ts @@ -0,0 +1,983 @@ +import fs from 'fs'; +import path from 'path'; +import crypto from 'crypto'; +import yaml from 'js-yaml'; +import { Types } from 'mongoose'; +import { + logger, + partitionIssues, + validateSkillName, + validateSkillBody, + validateRelativePath, + inferSkillFileCategory, + validateSkillFrontmatter, + validateSkillDescription, + deriveStructuredFrontmatterFields, +} from '@librechat/data-schemas'; +import type { ValidationIssue } from '@librechat/data-schemas'; +import type { CodeEnvRef } from 'librechat-data-provider'; +import { parseFrontmatter, guessMimeType } from './import'; + +export const DEPLOYMENT_SKILLS_DIR_ENV = 'DEPLOYMENT_SKILLS_DIR'; +export const DEFAULT_DEPLOYMENT_SKILLS_DIR = 'skill'; +export const DEPLOYMENT_SKILL_SOURCE = 'deployment'; +export const DEPLOYMENT_SKILL_FILE_SOURCE = 'deployment'; + +const SKILL_MD = 'SKILL.md'; +const DEPLOYMENT_AUTHOR_ID = new Types.ObjectId('de9100000000000000000000'); +const MAX_CACHED_TEXT_BYTES = 512 * 1024; + +type SkillId = Types.ObjectId | string; + +export type DeploymentSkillFile = { + _id: Types.ObjectId; + skillId: Types.ObjectId; + relativePath: string; + file_id: string; + filename: string; + filepath: string; + source: typeof DEPLOYMENT_SKILL_FILE_SOURCE; + mimeType: string; + bytes: number; + category: 'script' | 'reference' | 'asset' | 'other'; + isExecutable: boolean; + author: Types.ObjectId; + content?: string; + isBinary?: boolean; + codeEnvRef?: CodeEnvRef; + createdAt: Date; + updatedAt: Date; +}; + +export type DeploymentSkill = { + _id: Types.ObjectId; + name: string; + displayTitle?: string; + description: string; + body: string; + frontmatter: Record; + category: string; + disableModelInvocation?: boolean; + userInvocable?: boolean; + allowedTools?: string[]; + author: Types.ObjectId; + authorName: string; + version: number; + source: typeof DEPLOYMENT_SKILL_SOURCE; + sourceMetadata: { deployment: true; directory: string }; + fileCount: number; + alwaysApply: boolean; + isPublic: true; + deployment: true; + files: DeploymentSkillFile[]; + createdAt: Date; + updatedAt: Date; +}; + +type SkillLookupOptions = { + preferUserInvocable?: boolean; + preferModelInvocable?: boolean; +}; + +type SkillSummaryRow = { + _id: Types.ObjectId; + name: string; + displayTitle?: string; + description: string; + category?: string; + disableModelInvocation?: boolean; + userInvocable?: boolean; + allowedTools?: string[]; + author: Types.ObjectId; + authorName?: string; + version?: number; + source?: string; + sourceMetadata?: Record; + fileCount?: number; + alwaysApply?: boolean; + isPublic?: boolean; + tenantId?: string; + deployment?: boolean; + createdAt?: Date; + updatedAt?: Date; +}; + +type SkillDetailRow = SkillSummaryRow & { + body: string; + frontmatter?: Record; + version: number; + fileCount: number; +}; + +type AlwaysApplySkillRow = { + _id: Types.ObjectId; + name: string; + body: string; + author: Types.ObjectId | string; + allowedTools?: string[]; + deployment?: boolean; + updatedAt?: Date; +}; + +type ListSkillsByAccessParams = { + accessibleIds: Types.ObjectId[]; + category?: string; + search?: string; + limit: number; + cursor?: string | null; +}; + +type ListSkillsByAccessResult = { + skills: SkillSummaryRow[]; + has_more?: boolean; + after?: string | null; +}; + +type ListAlwaysApplyParams = { + accessibleIds: Types.ObjectId[]; + limit: number; + cursor?: string | null; +}; + +type ListAlwaysApplyResult = { + skills: AlwaysApplySkillRow[]; + has_more?: boolean; + after?: string | null; +}; + +type SkillFileRow = Omit & { + storageKey?: string; + storageRegion?: string; + tenantId?: string; +}; + +type SkillFileContentRow = SkillFileRow & { + codeEnvRef?: CodeEnvRef; + content?: string; + isBinary?: boolean; +}; + +export type DeploymentSkillBaseMethods = { + getSkillById?: (id: SkillId) => Promise; + getSkillByName?: ( + name: string, + accessibleIds: Types.ObjectId[], + options?: SkillLookupOptions, + ) => Promise; + listSkillsByAccess?: (params: ListSkillsByAccessParams) => Promise; + listAlwaysApplySkills?: (params: ListAlwaysApplyParams) => Promise; + listSkillFiles?: (skillId: SkillId) => Promise; + getSkillFileByPath?: ( + skillId: SkillId, + relativePath: string, + ) => Promise; + updateSkillFileContent?: ( + skillId: SkillId, + relativePath: string, + update: { content?: string; isBinary?: boolean }, + ) => Promise; + updateSkillFileCodeEnvIds?: ( + updates: Array<{ skillId: SkillId; relativePath: string; codeEnvRef: CodeEnvRef }>, + ) => Promise<{ matchedCount: number; modifiedCount: number } | void>; +}; + +type Cursor = { updatedAt: Date; _id: Types.ObjectId }; + +type LoadDeploymentSkillsOptions = { + projectRoot?: string; + env?: NodeJS.ProcessEnv; +}; + +type DirectoryResolution = { + directory: string; + explicitlyConfigured: boolean; +}; + +type LoadedSkillDirectory = { + directory: string; + relativeDirectory: string; +}; + +export class DeploymentSkillRegistry { + private readonly skillsById = new Map(); + private readonly skillsByName = new Map(); + private readonly filesByPath = new Map(); + + constructor( + private readonly directory: string | null, + skills: DeploymentSkill[], + ) { + for (const skill of skills) { + this.skillsById.set(skill._id.toString(), skill); + this.skillsByName.set(skill.name, skill); + for (const file of skill.files) { + this.filesByPath.set(file.filepath, file); + } + } + } + + getDirectory(): string | null { + return this.directory; + } + + list(): DeploymentSkill[] { + return Array.from(this.skillsById.values()); + } + + ids(): Types.ObjectId[] { + return this.list().map((skill) => skill._id); + } + + hasId(id: SkillId | undefined): boolean { + return id != null && this.skillsById.has(id.toString()); + } + + getById(id: SkillId): DeploymentSkill | null { + return this.skillsById.get(id.toString()) ?? null; + } + + getByName( + name: string, + accessibleIds: Types.ObjectId[], + _options?: SkillLookupOptions, + ): DeploymentSkill | null { + const skill = this.skillsByName.get(name); + if (!skill) { + return null; + } + if (!hasAccessibleId(accessibleIds, skill._id)) { + return null; + } + return skill; + } + + listByAccess(params: ListSkillsByAccessParams): DeploymentSkill[] { + const accessibleSet = new Set(params.accessibleIds.map((id) => id.toString())); + const cursor = decodeCursor(params.cursor); + const search = params.search?.toLowerCase(); + return this.list() + .filter((skill) => accessibleSet.has(skill._id.toString())) + .filter((skill) => !params.category || skill.category === params.category) + .filter((skill) => { + if (!search) { + return true; + } + return ( + skill.name.toLowerCase().includes(search) || + skill.description.toLowerCase().includes(search) || + (skill.displayTitle?.toLowerCase().includes(search) ?? false) + ); + }) + .filter((skill) => isAfterCursor(skill, cursor)) + .sort(compareBySkillCursor); + } + + listAlwaysApply(params: ListAlwaysApplyParams): DeploymentSkill[] { + const accessibleSet = new Set(params.accessibleIds.map((id) => id.toString())); + const cursor = decodeCursor(params.cursor); + return this.list() + .filter((skill) => skill.alwaysApply === true) + .filter((skill) => accessibleSet.has(skill._id.toString())) + .filter((skill) => isAfterCursor(skill, cursor)) + .sort(compareBySkillCursor); + } + + listFiles(skillId: SkillId): DeploymentSkillFile[] | null { + const skill = this.getById(skillId); + return skill ? [...skill.files] : null; + } + + getFileByPath(skillId: SkillId, relativePath: string): DeploymentSkillFile | null { + const skill = this.getById(skillId); + if (!skill) { + return null; + } + return skill.files.find((file) => file.relativePath === relativePath) ?? null; + } + + hasFilePath(filepath: string): boolean { + return this.filesByPath.has(filepath); + } + + updateFileCodeEnvRefs( + updates: Array<{ skillId: SkillId; relativePath: string; codeEnvRef: CodeEnvRef }>, + ): Array<{ skillId: SkillId; relativePath: string; codeEnvRef: CodeEnvRef }> { + const dbUpdates: Array<{ skillId: SkillId; relativePath: string; codeEnvRef: CodeEnvRef }> = []; + for (const update of updates) { + const file = this.getFileByPath(update.skillId, update.relativePath); + if (!file) { + dbUpdates.push(update); + continue; + } + file.codeEnvRef = update.codeEnvRef; + } + return dbUpdates; + } +} + +let registry = new DeploymentSkillRegistry(null, []); + +export function getDeploymentSkillRegistry(): DeploymentSkillRegistry { + return registry; +} + +export function getDeploymentSkillIds(): Types.ObjectId[] { + return registry.ids(); +} + +export function mergeDeploymentSkillIds(ids: Array): Types.ObjectId[] { + const seen = new Set(); + const merged: Types.ObjectId[] = []; + for (const id of [...ids, ...registry.ids()]) { + const oid = typeof id === 'string' ? new Types.ObjectId(id) : id; + const key = oid.toString(); + if (seen.has(key)) { + continue; + } + seen.add(key); + merged.push(oid); + } + return merged; +} + +export function isDeploymentSkillId(id: SkillId | undefined): boolean { + return registry.hasId(id); +} + +export function isDeploymentSkillFileSource(source: unknown): boolean { + return source === DEPLOYMENT_SKILL_FILE_SOURCE; +} + +export function getDeploymentSkillById(id: SkillId): DeploymentSkill | null { + return registry.getById(id); +} + +export function isDeploymentSkillFilePath(filepath: string): boolean { + return registry.hasFilePath(filepath); +} + +export async function getDeploymentSkillDownloadStream( + filepath: string, +): Promise { + if (!registry.hasFilePath(filepath)) { + throw new Error('Deployment skill file is not registered'); + } + return fs.createReadStream(filepath); +} + +export function updateDeploymentSkillFileCodeEnvRefs( + updates: Array<{ skillId: SkillId; relativePath: string; codeEnvRef: CodeEnvRef }>, +): Array<{ skillId: SkillId; relativePath: string; codeEnvRef: CodeEnvRef }> { + return registry.updateFileCodeEnvRefs(updates); +} + +export function resolveDeploymentSkillDirectory( + options: LoadDeploymentSkillsOptions = {}, +): DirectoryResolution { + const env = options.env ?? process.env; + const projectRoot = options.projectRoot ?? process.cwd(); + const configured = env[DEPLOYMENT_SKILLS_DIR_ENV]?.trim(); + const rawDirectory = + configured && configured.length > 0 ? configured : DEFAULT_DEPLOYMENT_SKILLS_DIR; + return { + directory: path.isAbsolute(rawDirectory) + ? rawDirectory + : path.resolve(projectRoot, rawDirectory), + explicitlyConfigured: configured != null && configured.length > 0, + }; +} + +export async function initializeDeploymentSkills( + options: LoadDeploymentSkillsOptions = {}, +): Promise { + const resolved = resolveDeploymentSkillDirectory(options); + registry = await loadDeploymentSkillsFromDirectory(resolved.directory, { + projectRoot: options.projectRoot ?? process.cwd(), + explicitlyConfigured: resolved.explicitlyConfigured, + }); + const count = registry.list().length; + if (count > 0) { + logger.info( + `[deploymentSkills] Loaded ${count} deployment skill(s) from ${registry.getDirectory()}`, + ); + } else { + logger.debug(`[deploymentSkills] No deployment skills loaded from ${resolved.directory}`); + } + return registry; +} + +export async function loadDeploymentSkillsFromDirectory( + directory: string, + options: { projectRoot?: string; explicitlyConfigured?: boolean } = {}, +): Promise { + let rootStat: fs.Stats; + try { + rootStat = await fs.promises.stat(directory); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code === 'ENOENT' && + options.explicitlyConfigured !== true + ) { + return new DeploymentSkillRegistry(directory, []); + } + throw new Error(`Deployment skills directory not found: ${directory}`); + } + if (!rootStat.isDirectory()) { + throw new Error(`Deployment skills path must be a directory: ${directory}`); + } + + const skillDirectories = await findSkillDirectories(directory, options.projectRoot ?? directory); + const skills = await Promise.all( + skillDirectories.map((skillDirectory) => loadDeploymentSkill(skillDirectory, directory)), + ); + validateUniqueNames(skills); + return new DeploymentSkillRegistry(directory, skills.sort(compareBySkillCursor)); +} + +export function createDeploymentSkillMethods( + base: T, +): T & Required> { + const methods = { + ...base, + getSkillById: async (id: SkillId): Promise => { + const deployment = registry.getById(id); + if (deployment) { + return toSkillDetailRow(deployment); + } + return base.getSkillById ? base.getSkillById(id) : null; + }, + getSkillByName: async ( + name: string, + accessibleIds: Types.ObjectId[], + options?: SkillLookupOptions, + ): Promise => { + const dbSkill = base.getSkillByName + ? await base.getSkillByName(name, stripDeploymentIds(accessibleIds), options) + : null; + const deployment = registry.getByName(name, accessibleIds, options); + return pickPreferredSkill( + [dbSkill, deployment ? toSkillDetailRow(deployment) : null], + options, + ); + }, + listSkillsByAccess: async ( + params: ListSkillsByAccessParams, + ): Promise => { + const dbResult = base.listSkillsByAccess + ? await base.listSkillsByAccess({ + ...params, + accessibleIds: stripDeploymentIds(params.accessibleIds), + }) + : { skills: [], has_more: false, after: null }; + return mergeSkillPage({ + dbResult, + deploymentRows: registry.listByAccess(params).map(toSkillSummaryRow), + limit: params.limit, + }); + }, + listAlwaysApplySkills: async ( + params: ListAlwaysApplyParams, + ): Promise => { + const dbResult = base.listAlwaysApplySkills + ? await base.listAlwaysApplySkills({ + ...params, + accessibleIds: stripDeploymentIds(params.accessibleIds), + }) + : { skills: [], has_more: false, after: null }; + return mergeAlwaysApplyPage({ + dbResult, + deploymentRows: registry.listAlwaysApply(params).map(toAlwaysApplyRow), + limit: params.limit, + }); + }, + listSkillFiles: async (skillId: SkillId): Promise => { + const deploymentFiles = registry.listFiles(skillId); + if (deploymentFiles) { + return deploymentFiles.map(toSkillFileRow); + } + return base.listSkillFiles ? base.listSkillFiles(skillId) : []; + }, + getSkillFileByPath: async ( + skillId: SkillId, + relativePath: string, + ): Promise => { + const deploymentFile = registry.getFileByPath(skillId, relativePath); + if (deploymentFile) { + return toSkillFileContentRow(deploymentFile); + } + return base.getSkillFileByPath ? base.getSkillFileByPath(skillId, relativePath) : null; + }, + updateSkillFileContent: async ( + skillId: SkillId, + relativePath: string, + update: { content?: string; isBinary?: boolean }, + ): Promise => { + const deploymentFile = registry.getFileByPath(skillId, relativePath); + if (deploymentFile) { + if (update.content !== undefined) { + deploymentFile.content = update.content; + } + if (update.isBinary !== undefined) { + deploymentFile.isBinary = update.isBinary; + } + return; + } + if (base.updateSkillFileContent) { + await base.updateSkillFileContent(skillId, relativePath, update); + } + }, + updateSkillFileCodeEnvIds: async ( + updates: Array<{ skillId: SkillId; relativePath: string; codeEnvRef: CodeEnvRef }>, + ): Promise<{ matchedCount: number; modifiedCount: number } | void> => { + const dbUpdates = registry.updateFileCodeEnvRefs(updates); + if (dbUpdates.length === 0) { + return { matchedCount: updates.length, modifiedCount: updates.length }; + } + return base.updateSkillFileCodeEnvIds + ? base.updateSkillFileCodeEnvIds(dbUpdates) + : { matchedCount: 0, modifiedCount: 0 }; + }, + }; + return methods as T & Required>; +} + +async function findSkillDirectories( + directory: string, + projectRoot: string, +): Promise { + const directories: LoadedSkillDirectory[] = []; + if (await fileExists(path.join(directory, SKILL_MD))) { + directories.push({ + directory, + relativeDirectory: relativeToRoot(projectRoot, directory), + }); + } + + const entries = await fs.promises.readdir(directory, { withFileTypes: true }); + const childDirectories = entries + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(directory, entry.name)); + const childSkillChecks = await Promise.all( + childDirectories.map(async (child) => + (await fileExists(path.join(child, SKILL_MD))) + ? { + directory: child, + relativeDirectory: relativeToRoot(projectRoot, child), + } + : null, + ), + ); + for (const child of childSkillChecks) { + if (child) { + directories.push(child); + } + } + return directories; +} + +async function loadDeploymentSkill( + skillDirectory: LoadedSkillDirectory, + rootDirectory: string, +): Promise { + const skillMdPath = path.join(skillDirectory.directory, SKILL_MD); + const [content, stat] = await Promise.all([ + fs.promises.readFile(skillMdPath, 'utf8'), + fs.promises.stat(skillMdPath), + ]); + const parsed = parseFrontmatter(content); + const structured = parseStructuredFrontmatter(content); + if ('error' in structured) { + throw new Error(`${skillDirectory.relativeDirectory}/${SKILL_MD}: ${structured.error}`); + } + const frontmatter = structured.frontmatter ?? {}; + const description = + typeof frontmatter.description === 'string' ? frontmatter.description : parsed.description; + const name = typeof frontmatter.name === 'string' ? frontmatter.name : parsed.name; + const issues: ValidationIssue[] = [ + ...validateSkillName(name), + ...validateSkillDescription(description), + ...validateSkillBody(content), + ...validateSkillFrontmatter(frontmatter), + ]; + if (parsed.invalidBooleans.length > 0) { + issues.push( + ...parsed.invalidBooleans.map((key) => ({ + field: `frontmatter.${key}`, + code: 'INVALID_TYPE', + message: `"${key}" must be a boolean (true or false)`, + })), + ); + } + const { errors, warnings } = partitionIssues(issues); + if (errors.length > 0) { + throw new Error( + `${skillDirectory.relativeDirectory}/${SKILL_MD}: ${errors + .map((issue) => `${issue.field}: ${issue.message}`) + .join('; ')}`, + ); + } + if (warnings.length > 0) { + logger.warn( + `[deploymentSkills] ${skillDirectory.relativeDirectory}/${SKILL_MD}: ${warnings + .map((issue) => `${issue.field}: ${issue.message}`) + .join('; ')}`, + ); + } + + const derived = deriveStructuredFrontmatterFields(frontmatter); + const skillId = stableObjectId(`deployment-skill:${name}`); + const files = await loadDeploymentSkillFiles({ + skillId, + skillName: name, + skillDirectory: skillDirectory.directory, + rootDirectory, + }); + return { + _id: skillId, + name, + description, + body: content, + frontmatter, + category: '', + author: DEPLOYMENT_AUTHOR_ID, + authorName: 'Deployment', + version: 1, + source: DEPLOYMENT_SKILL_SOURCE, + sourceMetadata: { + deployment: true, + directory: skillDirectory.relativeDirectory, + }, + fileCount: files.length, + alwaysApply: parsed.alwaysApply ?? false, + isPublic: true, + deployment: true, + files, + createdAt: stat.birthtime, + updatedAt: stat.mtime, + ...derived, + }; +} + +async function loadDeploymentSkillFiles({ + skillId, + skillName, + skillDirectory, + rootDirectory, +}: { + skillId: Types.ObjectId; + skillName: string; + skillDirectory: string; + rootDirectory: string; +}): Promise { + const files = await collectSkillFiles(skillDirectory, skillDirectory); + const rows = await Promise.all( + files.map(async (filePath) => { + const relativePath = normalizePath(path.relative(skillDirectory, filePath)); + const issues = validateRelativePath(relativePath); + if (issues.length > 0) { + throw new Error( + `${relativeToRoot(rootDirectory, filePath)}: ${issues + .map((issue) => `${issue.field}: ${issue.message}`) + .join('; ')}`, + ); + } + const stat = await fs.promises.stat(filePath); + const mimeType = guessMimeType(relativePath); + const cache = await readCachedFileContent(filePath, stat.size); + return { + _id: stableObjectId(`deployment-skill-file:${skillName}:${relativePath}`), + skillId, + relativePath, + file_id: stableObjectId(`deployment-skill-file-id:${skillName}:${relativePath}`).toString(), + filename: path.basename(relativePath), + filepath: filePath, + source: DEPLOYMENT_SKILL_FILE_SOURCE, + mimeType, + bytes: stat.size, + category: inferSkillFileCategory(relativePath), + isExecutable: false, + author: DEPLOYMENT_AUTHOR_ID, + createdAt: stat.birthtime, + updatedAt: stat.mtime, + ...cache, + } satisfies DeploymentSkillFile; + }), + ); + return rows.sort((a, b) => a.relativePath.localeCompare(b.relativePath)); +} + +async function collectSkillFiles(root: string, directory: string): Promise { + const entries = await fs.promises.readdir(directory, { withFileTypes: true }); + const results = await Promise.all( + entries.map(async (entry) => { + const filePath = path.join(directory, entry.name); + const relativePath = normalizePath(path.relative(root, filePath)); + if (relativePath === SKILL_MD) { + return []; + } + if (entry.isSymbolicLink()) { + throw new Error(`${relativePath}: symlinks are not allowed in deployment skills`); + } + if (entry.isDirectory()) { + return collectSkillFiles(root, filePath); + } + if (!entry.isFile()) { + return []; + } + return [filePath]; + }), + ); + return results.flat(); +} + +async function readCachedFileContent( + filePath: string, + bytes: number, +): Promise> { + const file = await fs.promises.open(filePath, 'r'); + try { + const probe = Buffer.alloc(Math.min(bytes, 8192)); + if (probe.length > 0) { + await file.read(probe, 0, probe.length, 0); + } + if (probe.includes(0)) { + return { isBinary: true }; + } + } finally { + await file.close(); + } + if (bytes > MAX_CACHED_TEXT_BYTES) { + return { isBinary: false }; + } + return { + isBinary: false, + content: await fs.promises.readFile(filePath, 'utf8'), + }; +} + +function parseStructuredFrontmatter( + content: string, +): { frontmatter?: Record; error?: undefined } | { error: string } { + const trimmed = content.trim(); + if (!trimmed.startsWith('---')) { + return { frontmatter: {} }; + } + const after = trimmed.slice(3); + const closingIdx = after.indexOf('\n---'); + if (closingIdx === -1) { + return { error: `Invalid ${SKILL_MD} frontmatter: missing closing "---".` }; + } + try { + const parsed = yaml.load(after.slice(0, closingIdx)); + if (parsed == null) { + return { frontmatter: {} }; + } + if (typeof parsed !== 'object' || Array.isArray(parsed)) { + return { error: `${SKILL_MD} frontmatter must be a YAML mapping.` }; + } + return { frontmatter: parsed as Record }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { error: `Invalid ${SKILL_MD} frontmatter: ${message}` }; + } +} + +function validateUniqueNames(skills: DeploymentSkill[]): void { + const seen = new Set(); + for (const skill of skills) { + if (seen.has(skill.name)) { + throw new Error(`Duplicate deployment skill name "${skill.name}"`); + } + seen.add(skill.name); + } +} + +function mergeSkillPage({ + dbResult, + deploymentRows, + limit, +}: { + dbResult: ListSkillsByAccessResult; + deploymentRows: SkillSummaryRow[]; + limit: number; +}): ListSkillsByAccessResult { + const boundedLimit = Math.min(Math.max(1, limit || 20), 100); + const merged = [...deploymentRows, ...dbResult.skills].sort(compareBySkillCursor); + const sliced = merged.slice(0, boundedLimit); + const hasMore = merged.length > boundedLimit || dbResult.has_more === true; + return { + skills: sliced, + has_more: hasMore, + after: hasMore && sliced.length > 0 ? encodeCursor(sliced[sliced.length - 1]) : null, + }; +} + +function mergeAlwaysApplyPage({ + dbResult, + deploymentRows, + limit, +}: { + dbResult: ListAlwaysApplyResult; + deploymentRows: AlwaysApplySkillRow[]; + limit: number; +}): ListAlwaysApplyResult { + const boundedLimit = Math.min(Math.max(1, limit || 20), 100); + const merged = [...deploymentRows, ...dbResult.skills].sort(compareBySkillCursor); + const sliced = merged.slice(0, boundedLimit); + const hasMore = merged.length > boundedLimit || dbResult.has_more === true; + return { + skills: sliced, + has_more: hasMore, + after: hasMore && sliced.length > 0 ? encodeCursor(sliced[sliced.length - 1]) : null, + }; +} + +function pickPreferredSkill( + rows: Array, + options?: SkillLookupOptions, +): SkillDetailRow | null { + const candidates = rows.filter((row): row is SkillDetailRow => row != null); + if (candidates.length === 0) { + return null; + } + const preferred = candidates.filter((row) => matchesLookupPreference(row, options)); + return (preferred.length > 0 ? preferred : candidates).sort(compareBySkillCursor)[0]; +} + +function matchesLookupPreference( + row: Pick, + options?: SkillLookupOptions, +): boolean { + if (options?.preferUserInvocable === true && row.userInvocable === false) { + return false; + } + if (options?.preferModelInvocable === true && row.disableModelInvocation === true) { + return false; + } + return true; +} + +function toSkillSummaryRow(skill: DeploymentSkill): SkillSummaryRow { + const { body: _body, frontmatter: _frontmatter, files: _files, ...row } = skill; + return row; +} + +function toSkillDetailRow(skill: DeploymentSkill): SkillDetailRow { + const { files: _files, ...row } = skill; + return row; +} + +function toAlwaysApplyRow(skill: DeploymentSkill): AlwaysApplySkillRow { + return { + _id: skill._id, + name: skill.name, + body: skill.body, + author: skill.author, + deployment: true, + updatedAt: skill.updatedAt, + ...(skill.allowedTools !== undefined ? { allowedTools: skill.allowedTools } : {}), + }; +} + +function toSkillFileRow(file: DeploymentSkillFile): SkillFileRow { + const { codeEnvRef: _codeEnvRef, content: _content, isBinary: _isBinary, ...row } = file; + return row; +} + +function toSkillFileContentRow(file: DeploymentSkillFile): SkillFileContentRow { + return { + ...toSkillFileRow(file), + codeEnvRef: file.codeEnvRef, + content: file.content, + isBinary: file.isBinary, + }; +} + +function stripDeploymentIds(ids: Types.ObjectId[]): Types.ObjectId[] { + return ids.filter((id) => !registry.hasId(id)); +} + +function hasAccessibleId(ids: Types.ObjectId[], skillId: Types.ObjectId): boolean { + const key = skillId.toString(); + return ids.some((id) => id.toString() === key); +} + +function compareBySkillCursor( + a: Pick, + b: Pick, +): number { + const aTime = (a.updatedAt ?? new Date(0)).getTime(); + const bTime = (b.updatedAt ?? new Date(0)).getTime(); + if (aTime !== bTime) { + return bTime - aTime; + } + return a._id.toString().localeCompare(b._id.toString()); +} + +function isAfterCursor(row: Pick, cursor: Cursor | null) { + if (!cursor) { + return true; + } + const rowTime = (row.updatedAt ?? new Date(0)).getTime(); + const cursorTime = cursor.updatedAt.getTime(); + if (rowTime < cursorTime) { + return true; + } + if (rowTime > cursorTime) { + return false; + } + return row._id.toString() > cursor._id.toString(); +} + +function decodeCursor(cursor: string | null | undefined): Cursor | null { + if (!cursor || cursor === 'undefined' || cursor === 'null') { + return null; + } + try { + const decoded = JSON.parse(Buffer.from(cursor, 'base64').toString('utf8')) as { + updatedAt?: string; + _id?: string; + }; + if (!decoded.updatedAt || !decoded._id || !Types.ObjectId.isValid(decoded._id)) { + return null; + } + const updatedAt = new Date(decoded.updatedAt); + if (Number.isNaN(updatedAt.getTime())) { + return null; + } + return { updatedAt, _id: new Types.ObjectId(decoded._id) }; + } catch { + return null; + } +} + +function encodeCursor(row: Pick): string { + return Buffer.from( + JSON.stringify({ + updatedAt: (row.updatedAt ?? new Date(0)).toISOString(), + _id: row._id.toString(), + }), + ).toString('base64'); +} + +function stableObjectId(seed: string): Types.ObjectId { + return new Types.ObjectId(crypto.createHash('sha1').update(seed).digest('hex').slice(0, 24)); +} + +async function fileExists(filePath: string): Promise { + try { + const stat = await fs.promises.stat(filePath); + return stat.isFile(); + } catch { + return false; + } +} + +function normalizePath(value: string): string { + return value.split(path.sep).join('/'); +} + +function relativeToRoot(root: string, target: string): string { + const relative = normalizePath(path.relative(root, target)); + return relative.length > 0 ? relative : '.'; +} diff --git a/packages/api/src/skills/import.ts b/packages/api/src/skills/import.ts index c5d7493277..190abf3b3d 100644 --- a/packages/api/src/skills/import.ts +++ b/packages/api/src/skills/import.ts @@ -679,6 +679,6 @@ const MIME_MAP: Record = { '.pdf': 'application/pdf', }; -function guessMimeType(filename: string): string { +export function guessMimeType(filename: string): string { return MIME_MAP[path.extname(filename).toLowerCase()] || 'application/octet-stream'; } diff --git a/packages/api/src/skills/index.ts b/packages/api/src/skills/index.ts index 5d32c38cbe..affee245c4 100644 --- a/packages/api/src/skills/index.ts +++ b/packages/api/src/skills/index.ts @@ -2,3 +2,4 @@ export * from './binary'; export * from './handlers'; export * from './import'; export * from './skillStates'; +export * from './deployment'; diff --git a/packages/data-provider/src/types/skills.ts b/packages/data-provider/src/types/skills.ts index 5d6e86a041..302c865213 100644 --- a/packages/data-provider/src/types/skills.ts +++ b/packages/data-provider/src/types/skills.ts @@ -25,9 +25,11 @@ export const SKILL_NAME_PATTERN = /^[a-z0-9][a-z0-9-]*$/; /** * Source of a skill — where its canonical definition came from. * `inline` means the skill was authored directly in LibreChat. + * `deployment` means the skill was loaded from the server's configured + * deployment skill directory and is not persisted as a Skill document. * `github` / `notion` are reserved for future sync integrations. */ -export type SkillSource = 'inline' | 'github' | 'notion'; +export type SkillSource = 'inline' | 'deployment' | 'github' | 'notion'; /** * Category inferred from a skill file's top-level directory prefix. @@ -92,8 +94,8 @@ export type TSkillWarning = { * - `frontmatter` is the structured YAML bag minus `name`/`description` * (those live as top-level columns). Validated strictly against a known * key set server-side. - * - `source`/`sourceMetadata` are reserved for phase 2+ external sync and - * always `'inline'` / absent in phase 1. + * - `source`/`sourceMetadata` identify whether the row is user-authored, + * deployment-provided, or reserved for a future sync provider. */ export type TSkill = { _id: string; diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts index 8c95d7f9cc..32fb4aa85b 100644 --- a/packages/data-schemas/src/index.ts +++ b/packages/data-schemas/src/index.ts @@ -15,6 +15,14 @@ export { premiumTokenValues, defaultRate, permissionBitSupersets, + partitionIssues, + validateSkillName, + validateSkillBody, + validateRelativePath, + inferSkillFileCategory, + validateSkillFrontmatter, + validateSkillDescription, + deriveStructuredFrontmatterFields, } from './methods'; export type * from './types'; export type * from './methods'; diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts index 0663a7a819..abedc18e12 100644 --- a/packages/data-schemas/src/methods/index.ts +++ b/packages/data-schemas/src/methods/index.ts @@ -58,6 +58,14 @@ import { createSpendTokensMethods, type SpendTokensMethods } from './spendTokens import { createPromptMethods, type PromptMethods, type PromptDeps } from './prompt'; import { createSkillMethods, + partitionIssues, + validateSkillName, + validateSkillBody, + validateRelativePath, + validateSkillFrontmatter, + validateSkillDescription, + deriveStructuredFrontmatterFields, + inferSkillFileCategory, type SkillMethods, type SkillDeps, type CreateSkillInput, @@ -77,6 +85,16 @@ import { createConfigMethods, type ConfigMethods } from './config'; export { RoleConflictError, DEFAULT_REFRESH_TOKEN_EXPIRY, DEFAULT_SESSION_EXPIRY }; export { tokenValues, cacheTokenValues, premiumTokenValues, defaultRate }; export { permissionBitSupersets }; +export { + partitionIssues, + validateSkillName, + validateSkillBody, + validateRelativePath, + validateSkillFrontmatter, + validateSkillDescription, + deriveStructuredFrontmatterFields, + inferSkillFileCategory, +}; export type AllMethods = UserMethods & SessionMethods & diff --git a/skill/README.md b/skill/README.md new file mode 100644 index 0000000000..42da54931b --- /dev/null +++ b/skill/README.md @@ -0,0 +1,15 @@ +# Deployment Skills + +Place shared deployment skills in this directory. Each skill should live in its own folder with a +`SKILL.md` file, for example: + +```text +skill/ + my-shared-skill/ + SKILL.md + references/ + notes.md +``` + +These skills are loaded at server startup, exposed read-only to all users with Skills enabled, and +are not persisted as Skill documents in MongoDB.