🧭 feat: Scope Model Spec Skills (#13522)

* feat: scope model spec skills

* style: format skill catalog limit

* fix: serialize model spec skill resolution

* test: satisfy model spec load config typing

* fix: apply model spec skills to added conversations

* fix: support alwaysApply frontmatter alias

* fix: address model spec skills review

e2e/config/librechat.e2e.yaml

@@ -25,3 +25,27 @@ endpoints:
         fetch: false
       titleConvo: false
       modelDisplayLabel: 'Mock Provider B'
+
+modelSpecs:
+  list:
+    - name: 'e2e-mock-provider-a'
+      label: 'Mock Provider A'
+      preset:
+        endpoint: 'Mock Provider A'
+        model: 'mock-model-a'
+
+    - name: 'e2e-mock-provider-b'
+      label: 'Mock Provider B'
+      preset:
+        endpoint: 'Mock Provider B'
+        model: 'mock-model-b'
+
+    - name: 'e2e-skill-scope'
+      label: 'E2E Skill Scope'
+      preset:
+        endpoint: 'Mock Provider A'
+        model: 'mock-model-a'
+      skills:
+        - 'e2e-model-spec-allowed'
+        - 'e2e-model-spec-missing'
+        - 'e2e-model-spec-inaccessible'

e2e/setup/fake-model.js

@@ -14,13 +14,20 @@ const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;
 
 const CREATE_SKILL_MARKER = 'E2E_CREATE_SKILL:';
 const EDIT_SKILL_MARKER = 'E2E_EDIT_SKILL:';
+const ASSERT_MODEL_SPEC_SKILLS_MARKER = 'E2E_ASSERT_MODEL_SPEC_SKILLS';
 const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
 const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
+const MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed';
 const CREATE_FILE_TOOL_NAME = 'create_file';
 const EDIT_FILE_TOOL_NAME = 'edit_file';
 const BASH_TOOL_NAME = 'bash_tool';
+const SKILL_TOOL_NAME = 'skill';
 const CREATE_SKILL_TOOL_CALL_ID = 'call_e2e_create_skill';
 const EDIT_SKILL_TOOL_CALL_ID = 'call_e2e_edit_skill';
+const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed';
+const MODEL_SPEC_MISSING_SKILL = 'e2e-model-spec-missing';
+const MODEL_SPEC_INACCESSIBLE_SKILL = 'e2e-model-spec-inaccessible';
+const ALWAYS_APPLY_BODY_MARKER = 'E2E_ALWAYS_APPLY_BODY_MARKER';
 const SKILL_DESCRIPTION =
   'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.';
 const EDITED_SKILL_DESCRIPTION =
@@ -108,6 +115,67 @@ function collectToolNames(agents) {
   return names;
 }
 
+function collectAdditionalInstructions(agents) {
+  return (agents ?? [])
+    .map((agent) =>
+      typeof agent?.additional_instructions === 'string' ? agent.additional_instructions : '',
+    )
+    .filter(Boolean)
+    .join('\n');
+}
+
+function collectSkillPrimeMessages(messages) {
+  return (messages ?? [])
+    .filter((message) => message?.additional_kwargs?.source === 'skill')
+    .map((message) => ({
+      name: message.additional_kwargs.skillName,
+      trigger: message.additional_kwargs.trigger,
+      content: getContentText(message.content),
+    }));
+}
+
+function modelSpecSkillAssertionResponses({ agents, messages, toolNames }) {
+  const failures = [];
+  const additionalInstructions = collectAdditionalInstructions(agents);
+  const skillPrimeMessages = collectSkillPrimeMessages(messages);
+  const alwaysApplyPrime = skillPrimeMessages.find(
+    (message) => message.name === MODEL_SPEC_ACCESSIBLE_SKILL && message.trigger === 'always-apply',
+  );
+
+  if (!toolNames.has(SKILL_TOOL_NAME)) {
+    failures.push(`${SKILL_TOOL_NAME} tool was not advertised`);
+  }
+  if (!additionalInstructions.includes(MODEL_SPEC_ACCESSIBLE_SKILL)) {
+    failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} was not present in the model-visible catalog`);
+  }
+  if (additionalInstructions.includes(MODEL_SPEC_MISSING_SKILL)) {
+    failures.push(`${MODEL_SPEC_MISSING_SKILL} leaked into the model-visible catalog`);
+  }
+  if (additionalInstructions.includes(MODEL_SPEC_INACCESSIBLE_SKILL)) {
+    failures.push(`${MODEL_SPEC_INACCESSIBLE_SKILL} leaked into the model-visible catalog`);
+  }
+  if (!alwaysApplyPrime) {
+    failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} was not always-apply primed`);
+  } else if (!alwaysApplyPrime.content.includes(ALWAYS_APPLY_BODY_MARKER)) {
+    failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} always-apply body was missing its marker`);
+  }
+  if (skillPrimeMessages.some((message) => message.name === MODEL_SPEC_MISSING_SKILL)) {
+    failures.push(`${MODEL_SPEC_MISSING_SKILL} was unexpectedly primed`);
+  }
+  if (skillPrimeMessages.some((message) => message.name === MODEL_SPEC_INACCESSIBLE_SKILL)) {
+    failures.push(`${MODEL_SPEC_INACCESSIBLE_SKILL} was unexpectedly primed`);
+  }
+
+  if (failures.length > 0) {
+    return {
+      responses: [`E2E model spec skill assertion failed: ${failures.join('; ')}`],
+    };
+  }
+  return {
+    responses: [`${MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT}: ${MODEL_SPEC_ACCESSIBLE_SKILL}`],
+  };
+}
+
 function buildSkillBody(skillName) {
   return `---
 name: ${skillName}
@@ -166,7 +234,11 @@ function fileAuthoringResponses(operation, toolNames) {
   };
 }
 
-function resolveResponses(text, toolNames) {
+function resolveResponses({ agents, messages, text, toolNames }) {
+  if (text.includes(ASSERT_MODEL_SPEC_SKILLS_MARKER)) {
+    return modelSpecSkillAssertionResponses({ agents, messages, toolNames });
+  }
+
   const createSkillName = getRequestedSkillName(text, CREATE_SKILL_MARKER);
   if (createSkillName) {
     return fileAuthoringResponses(
@@ -208,6 +280,11 @@ module.exports = function fakeModelHook(run, context) {
 
   const text = getLatestUserText(context?.messages);
   const toolNames = collectToolNames(context?.agents);
-  const { responses, toolCalls } = resolveResponses(text, toolNames);
+  const { responses, toolCalls } = resolveResponses({
+    agents: context?.agents,
+    messages: context?.messages,
+    text,
+    toolNames,
+  });
   graph.overrideTestModel(responses, CHUNK_DELAY_MS, toolCalls);
 };

e2e/specs/mock/helpers.ts

@@ -14,6 +14,10 @@ export type MockEndpoint = (typeof MOCK_ENDPOINTS)[number];
 
 export const NEW_CHAT_PATH = '/c/new';
 
+type RefreshTokenBody = {
+  token?: string;
+};
+
 export function isAgentsStream(response: Response) {
   return response.url().includes('/api/agents') && response.status() === 200;
 }
@@ -21,12 +25,30 @@ export function isAgentsStream(response: Response) {
 const modelSelectorTrigger = (page: Page) =>
   page.getByRole('button', { name: 'Select a model' }).first();
 
+const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
 /** Open the model selector, choose an endpoint, then its model (committed on the model click). */
 export async function selectMockEndpoint(page: Page, endpoint: MockEndpoint) {
-  await modelSelectorTrigger(page).click();
+  const trigger = modelSelectorTrigger(page);
+  await trigger.click();
   await page.getByRole('option', { name: endpoint.label }).click();
-  await page.getByRole('option', { name: endpoint.model, exact: true }).click();
-  await expect(modelSelectorTrigger(page)).not.toHaveText('Select a model');
+  const modelOption = page.getByRole('option', { name: endpoint.model, exact: true });
+  if (await modelOption.isVisible({ timeout: 1000 }).catch(() => false)) {
+    await modelOption.click();
+  }
+  await expect(trigger).not.toHaveText('Select a model');
+}
+
+/** Open the model selector and choose a configured model spec by label. */
+export async function selectModelSpec(page: Page, label: string) {
+  const trigger = modelSelectorTrigger(page);
+  await expect(trigger).toBeVisible();
+  if ((await trigger.textContent())?.includes(label)) {
+    return;
+  }
+  await trigger.click();
+  await page.getByRole('option', { name: new RegExp(`^${escapeRegExp(label)}\\b`) }).click();
+  await expect(trigger).toContainText(label);
 }
 
 /** Enable the ephemeral Skills capability from the composer tool menu. */
@@ -53,3 +75,88 @@ export async function sendMessage(page: Page, text: string): Promise<Response> {
   ]);
   return response;
 }
+
+export async function getAccessToken(page: Page): Promise<string> {
+  const result = await page.evaluate(async () => {
+    const response = await fetch('/api/auth/refresh', {
+      method: 'POST',
+      credentials: 'include',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({}),
+    });
+    const text = await response.text();
+    let json: unknown = null;
+    try {
+      json = text ? JSON.parse(text) : null;
+    } catch {
+      json = null;
+    }
+    return { ok: response.ok, status: response.status, text, json };
+  });
+
+  if (!result.ok) {
+    throw new Error(
+      `Expected /api/auth/refresh to return 2xx, got ${result.status}: ${result.text}`,
+    );
+  }
+
+  const body = result.json as RefreshTokenBody | null;
+  if (!body?.token) {
+    throw new Error(`Expected /api/auth/refresh to return a token, got: ${result.text}`);
+  }
+
+  return body.token;
+}
+
+export async function requestJson<T>(
+  page: Page,
+  params: {
+    path: string;
+    token: string;
+    method?: string;
+    body?: unknown;
+  },
+): Promise<T> {
+  const result = await page.evaluate(
+    async ({ accessToken, body, method, urlPath }) => {
+      const headers: Record<string, string> = {
+        Authorization: `Bearer ${accessToken}`,
+      };
+      const init: RequestInit = {
+        method,
+        credentials: 'include',
+        headers,
+      };
+      if (body !== undefined) {
+        headers['Content-Type'] = 'application/json';
+        init.body = JSON.stringify(body);
+      }
+      const response = await fetch(urlPath, init);
+      const text = await response.text();
+      let json: unknown = null;
+      try {
+        json = text ? JSON.parse(text) : null;
+      } catch {
+        json = null;
+      }
+      return { ok: response.ok, status: response.status, text, json };
+    },
+    {
+      accessToken: params.token,
+      body: params.body,
+      method: params.method ?? 'GET',
+      urlPath: params.path,
+    },
+  );
+
+  if (!result.ok) {
+    throw new Error(
+      `Expected ${params.method ?? 'GET'} ${params.path} to return 2xx, got ${result.status}: ${result.text}`,
+    );
+  }
+  return result.json as T;
+}
+
+export async function fetchJson<T>(page: Page, path: string, token: string): Promise<T> {
+  return requestJson<T>(page, { path, token });
+}

e2e/specs/mock/model-spec-skills.spec.ts

@@ -0,0 +1,175 @@
+import { expect, test } from '@playwright/test';
+import { MongoClient, ObjectId } from 'mongodb';
+import type { Page } from '@playwright/test';
+import { applyRuntimeEnv } from '../../setup/runtimeEnv';
+import {
+  NEW_CHAT_PATH,
+  fetchJson,
+  getAccessToken,
+  requestJson,
+  selectModelSpec,
+  sendMessage,
+} from './helpers';
+
+const MODEL_SPEC_LABEL = 'E2E Skill Scope';
+const ASSERTION_MARKER = 'E2E_ASSERT_MODEL_SPEC_SKILLS';
+const ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed';
+const ACCESSIBLE_SKILL_NAME = 'e2e-model-spec-allowed';
+const INACCESSIBLE_SKILL_NAME = 'e2e-model-spec-inaccessible';
+const ALWAYS_APPLY_BODY_MARKER = 'E2E_ALWAYS_APPLY_BODY_MARKER';
+const INACCESSIBLE_AUTHOR_ID = new ObjectId('64f000000000000000000001');
+const SKILL_DESCRIPTION =
+  'Use this skill to verify model-spec skill scoping and always-apply priming in mock e2e tests.';
+
+type SkillSummary = {
+  _id: string;
+  name: string;
+  description: string;
+  version: number;
+  alwaysApply?: boolean;
+};
+
+type SkillDetail = SkillSummary & {
+  body: string;
+};
+
+function buildSkillBody(name: string) {
+  return `---
+name: ${name}
+description: ${SKILL_DESCRIPTION}
+alwaysApply: true
+---
+
+# ${name}
+
+${ALWAYS_APPLY_BODY_MARKER}
+
+This body should be injected as an always-apply skill prime.`;
+}
+
+async function findSkill(
+  page: Page,
+  skillName: string,
+  token: string,
+): Promise<SkillSummary | null> {
+  const body = await fetchJson<{ skills?: SkillSummary[] }>(
+    page,
+    `/api/skills?search=${encodeURIComponent(skillName)}&limit=10`,
+    token,
+  );
+  return body.skills?.find((skill) => skill.name === skillName) ?? null;
+}
+
+async function seedAccessibleSkill(page: Page, token: string): Promise<SkillDetail> {
+  const body = buildSkillBody(ACCESSIBLE_SKILL_NAME);
+  const payload = {
+    name: ACCESSIBLE_SKILL_NAME,
+    description: SKILL_DESCRIPTION,
+    body,
+  };
+  const existing = await findSkill(page, ACCESSIBLE_SKILL_NAME, token);
+  if (!existing) {
+    return requestJson<SkillDetail>(page, {
+      path: '/api/skills',
+      token,
+      method: 'POST',
+      body: payload,
+    });
+  }
+
+  const detail = await fetchJson<SkillDetail>(
+    page,
+    `/api/skills/${encodeURIComponent(existing._id)}`,
+    token,
+  );
+  return requestJson<SkillDetail>(page, {
+    path: `/api/skills/${encodeURIComponent(existing._id)}`,
+    token,
+    method: 'PATCH',
+    body: {
+      ...payload,
+      expectedVersion: detail.version,
+    },
+  });
+}
+
+async function seedInaccessibleSkill() {
+  applyRuntimeEnv();
+  if (!process.env.MONGO_URI) {
+    throw new Error('MONGO_URI must be available for model-spec skill mock e2e tests');
+  }
+
+  const client = new MongoClient(process.env.MONGO_URI);
+  await client.connect();
+  try {
+    const db = client.db();
+    const skills = db.collection('skills');
+    const aclEntries = db.collection('aclentries');
+    const now = new Date();
+    await skills.updateOne(
+      {
+        name: INACCESSIBLE_SKILL_NAME,
+        author: INACCESSIBLE_AUTHOR_ID,
+        tenantId: null,
+      },
+      {
+        $set: {
+          description: SKILL_DESCRIPTION,
+          body: buildSkillBody(INACCESSIBLE_SKILL_NAME),
+          authorName: 'Inaccessible E2E User',
+          updatedAt: now,
+        },
+        $setOnInsert: {
+          displayTitle: INACCESSIBLE_SKILL_NAME,
+          frontmatter: {},
+          disableModelInvocation: false,
+          userInvocable: true,
+          allowedTools: [],
+          alwaysApply: true,
+          source: 'inline',
+          fileCount: 0,
+          version: 1,
+          createdAt: now,
+        },
+      },
+      { upsert: true },
+    );
+    const skill = await skills.findOne({
+      name: INACCESSIBLE_SKILL_NAME,
+      author: INACCESSIBLE_AUTHOR_ID,
+      tenantId: null,
+    });
+    if (skill?._id) {
+      await aclEntries.deleteMany({ resourceType: 'skill', resourceId: skill._id });
+    }
+  } finally {
+    await client.close();
+  }
+}
+
+test.describe('model spec skills', () => {
+  test('loads accessible configured skills and skips missing or inaccessible names', async ({
+    page,
+  }) => {
+    test.setTimeout(120000);
+
+    await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+    const token = await getAccessToken(page);
+    const skill = await seedAccessibleSkill(page, token);
+    expect(skill.alwaysApply).toBe(true);
+    await seedInaccessibleSkill();
+
+    await selectModelSpec(page, MODEL_SPEC_LABEL);
+    const response = await sendMessage(
+      page,
+      `${ASSERTION_MARKER}\nVerify model-spec skill scope and always-apply frontmatter.`,
+    );
+    expect(response.ok()).toBeTruthy();
+
+    await expect(
+      page
+        .getByTestId('messages-view')
+        .getByText(`${ASSERTION_FINAL_TEXT}: ${ACCESSIBLE_SKILL_NAME}`),
+    ).toBeVisible({ timeout: 30000 });
+  });
+});

e2e/specs/mock/skill-file-authoring.spec.ts

@@ -4,6 +4,8 @@ import {
   MOCK_ENDPOINTS,
   NEW_CHAT_PATH,
   enableSkills,
+  fetchJson,
+  getAccessToken,
   selectMockEndpoint,
   sendMessage,
 } from './helpers';
@@ -25,69 +27,8 @@ type SkillDetail = SkillSummary & {
   body: string;
 };
 
-type RefreshTokenBody = {
-  token?: string;
-};
-
 const uniqueSkillName = () => `e2e-file-authoring-${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
 
-async function getAccessToken(page: Page): Promise<string> {
-  const result = await page.evaluate(async () => {
-    const response = await fetch('/api/auth/refresh', {
-      method: 'POST',
-      credentials: 'include',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({}),
-    });
-    const text = await response.text();
-    let json: unknown = null;
-    try {
-      json = text ? JSON.parse(text) : null;
-    } catch {
-      json = null;
-    }
-    return { ok: response.ok, status: response.status, text, json };
-  });
-
-  if (!result.ok) {
-    throw new Error(
-      `Expected /api/auth/refresh to return 2xx, got ${result.status}: ${result.text}`,
-    );
-  }
-
-  const body = result.json as RefreshTokenBody | null;
-  if (!body?.token) {
-    throw new Error(`Expected /api/auth/refresh to return a token, got: ${result.text}`);
-  }
-
-  return body.token;
-}
-
-async function fetchJson<T>(page: Page, path: string, token: string): Promise<T> {
-  const result = await page.evaluate(
-    async ({ accessToken, urlPath }) => {
-      const response = await fetch(urlPath, {
-        credentials: 'include',
-        headers: { Authorization: `Bearer ${accessToken}` },
-      });
-      const text = await response.text();
-      let json: unknown = null;
-      try {
-        json = text ? JSON.parse(text) : null;
-      } catch {
-        json = null;
-      }
-      return { ok: response.ok, status: response.status, text, json };
-    },
-    { accessToken: token, urlPath: path },
-  );
-
-  if (!result.ok) {
-    throw new Error(`Expected ${path} to return 2xx, got ${result.status}: ${result.text}`);
-  }
-  return result.json as T;
-}
-
 async function findSkill(
   page: Page,
   skillName: string,