🧩 fix: Harden Agent Skill Lifecycles End to End (#14429)
Some checks failed
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Has been cancelled
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Has been cancelled
GitNexus Index / index (push) Has been cancelled
GitNexus Index / post-index (push) Has been cancelled

* test: cover agent skill lifecycles end to end

* style: sort agent skill imports
This commit is contained in:
Danny Avila 2026-07-25 08:19:12 -04:00 • committed by GitHub
parent 73699b5c25
commit f3159f9891
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 2160 additions and 116 deletions

View file

@ -10,7 +10,7 @@ const {
const { isEphemeralAgentId } = require('librechat-data-provider');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
const { getMCPServerTools } = require('~/server/services/Config');
const { canAuthorSkillFiles } = require('./skillDeps');
const { getSkillDbMethods, canAuthorSkillFiles } = require('./skillDeps');
const db = require('~/models');
const loadAddedAgent = (params) =>
@ -97,6 +97,7 @@ const processAddedConvo = async ({
});
try {
const skillDbMethods = getSkillDbMethods();
const addedAgent = await loadAddedAgent({ req, conversation: addedConvo, primaryAgent });
if (!addedAgent) {
return { userMCPAuthMap };
@ -138,7 +139,7 @@ const processAddedConvo = async ({
const resolvedSkillIds = await resolveModelSpecSkillIds({
names: selectedModelSpec.skills,
accessibleSkillIds,
getSkillByName: db.getSkillByName,
getSkillByName: skillDbMethods.getSkillByName,
});
addedAgent.skills_enabled = true;
addedAgent.skills = resolvedSkillIds.map((id) => id.toString());
@ -195,9 +196,9 @@ const processAddedConvo = async ({
getToolFilesByIds: db.getToolFilesByIds,
getCodeGeneratedFiles: db.getCodeGeneratedFiles,
filterFilesByAgentAccess,
listSkillsByAccess: db.listSkillsByAccess,
listAlwaysApplySkills: db.listAlwaysApplySkills,
getSkillByName: db.getSkillByName,
listSkillsByAccess: skillDbMethods.listSkillsByAccess,
listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills,
getSkillByName: skillDbMethods.getSkillByName,
},
);

View file

@ -4,8 +4,12 @@ const mockLoadAddedAgent = jest.fn();
const mockResolveAgentScopedSkillIds = jest.fn();
const mockResolveModelSpecSkillIds = jest.fn();
const mockCanAuthorSkillFiles = jest.fn();
const mockGetSkillDbMethods = jest.fn();
const mockGetAgent = jest.fn();
const mockGetMCPServerTools = jest.fn();
const mockRegistryGetSkillByName = jest.fn();
const mockRegistryListSkillsByAccess = jest.fn();
const mockRegistryListAlwaysApplySkills = jest.fn();
jest.mock('@librechat/data-schemas', () => ({
logger: {
@ -35,6 +39,7 @@ jest.mock('~/server/services/Config', () => ({
jest.mock('./skillDeps', () => ({
canAuthorSkillFiles: (...args) => mockCanAuthorSkillFiles(...args),
getSkillDbMethods: () => mockGetSkillDbMethods(),
}));
jest.mock('~/models', () => ({
@ -45,7 +50,6 @@ jest.mock('~/models', () => ({
}));
const { processAddedConvo } = require('./addedConvo');
const db = require('~/models');
const { Constants } = require('librechat-data-provider');
const makeReq = () => ({ user: { id: 'u1', role: 'USER' } });
@ -72,6 +76,11 @@ describe('processAddedConvo', () => {
);
mockResolveModelSpecSkillIds.mockResolvedValue([]);
mockCanAuthorSkillFiles.mockReturnValue(false);
mockGetSkillDbMethods.mockReturnValue({
getSkillByName: mockRegistryGetSkillByName,
listSkillsByAccess: mockRegistryListSkillsByAccess,
listAlwaysApplySkills: mockRegistryListAlwaysApplySkills,
});
});
const baseParams = (overrides = {}) => ({
@ -125,6 +134,75 @@ describe('processAddedConvo', () => {
);
});
it('keeps deployment-aware skill metadata on a persisted added-agent config', async () => {
const deploymentSkillId = { toString: () => 'deployment-skill' };
const agentConfigs = new Map();
const initializedConfig = {
id: 'persisted-added-agent',
additional_instructions: '<skill_catalog>deployment-skill</skill_catalog>',
manualSkillPrimes: [],
alwaysApplySkillPrimes: [
{
_id: 'deployment-skill',
name: 'deployment-skill',
body: 'deployment skill body',
},
],
toolDefinitions: [{ name: 'skill' }],
userMCPAuthMap: undefined,
};
mockLoadAddedAgent.mockResolvedValue({
id: 'persisted-added-agent',
provider: 'openai',
skills_enabled: true,
skills: ['deployment-skill'],
});
mockResolveAgentScopedSkillIds.mockReturnValue([deploymentSkillId]);
mockInitializeAgent.mockResolvedValue(initializedConfig);
await processAddedConvo(
baseParams({
accessibleSkillIds: [deploymentSkillId],
editableSkillIds: [deploymentSkillId],
skillsCapabilityEnabled: true,
agentConfigs,
}),
);
expect(mockResolveModelSpecSkillIds).not.toHaveBeenCalled();
expect(mockInitializeAgent).toHaveBeenCalledWith(
expect.objectContaining({
agent: expect.objectContaining({
id: 'persisted-added-agent',
skills_enabled: true,
skills: ['deployment-skill'],
}),
accessibleSkillIds: [deploymentSkillId],
}),
expect.objectContaining({
listSkillsByAccess: mockRegistryListSkillsByAccess,
listAlwaysApplySkills: mockRegistryListAlwaysApplySkills,
getSkillByName: mockRegistryGetSkillByName,
}),
);
expect(agentConfigs.get('persisted-added-agent')).toBe(initializedConfig);
expect(agentConfigs.get('persisted-added-agent')).toEqual(
expect.objectContaining({
additional_instructions: '<skill_catalog>deployment-skill</skill_catalog>',
manualSkillPrimes: [],
alwaysApplySkillPrimes: [
expect.objectContaining({
name: 'deployment-skill',
body: 'deployment skill body',
}),
],
toolDefinitions: [expect.objectContaining({ name: 'skill' })],
}),
);
expect(mockGetSkillDbMethods).toHaveBeenCalledTimes(1);
});
it('resolves and forwards model-spec skill scope for added ephemeral agents', async () => {
const accessibleSkillId = { toString: () => 'accessible-skill' };
const editableSkillId = { toString: () => 'editable-skill' };
@ -181,7 +259,7 @@ describe('processAddedConvo', () => {
expect(mockResolveModelSpecSkillIds).toHaveBeenCalledWith({
names: ['finance-analyst'],
accessibleSkillIds: [accessibleSkillId],
getSkillByName: db.getSkillByName,
getSkillByName: mockRegistryGetSkillByName,
});
expect(mockResolveAgentScopedSkillIds).toHaveBeenNthCalledWith(1, {
agent: expect.objectContaining({
@ -222,10 +300,11 @@ describe('processAddedConvo', () => {
defaultActiveOnShare: true,
}),
expect.objectContaining({
listSkillsByAccess: db.listSkillsByAccess,
listAlwaysApplySkills: db.listAlwaysApplySkills,
getSkillByName: db.getSkillByName,
listSkillsByAccess: mockRegistryListSkillsByAccess,
listAlwaysApplySkills: mockRegistryListAlwaysApplySkills,
getSkillByName: mockRegistryGetSkillByName,
}),
);
expect(mockGetSkillDbMethods).toHaveBeenCalledTimes(1);
});
});

View file

@ -402,7 +402,7 @@ const initializeClient = async ({ req, res, signal, endpointOption, jobCreatedAt
const resolvedSkillIds = await resolveModelSpecSkillIds({
names: selectedModelSpec.skills,
accessibleSkillIds,
getSkillByName: db.getSkillByName,
getSkillByName: skillDbMethods.getSkillByName,
});
primaryAgent.skills_enabled = true;
primaryAgent.skills = resolvedSkillIds.map((id) => id.toString());

View file

@ -79,7 +79,7 @@ jest.mock('~/cache', () => ({
}));
const { initializeClient } = require('./initialize');
const { getSkillToolDeps } = require('./skillDeps');
const { getSkillDbMethods, getSkillToolDeps } = require('./skillDeps');
const { getModelsConfig } = require('~/server/controllers/ModelController');
const { logger } = require('@librechat/data-schemas');
const { User, AclEntry } = require('~/db/models');
@ -360,6 +360,63 @@ describe('initializeClient — processAgent ACL gate', () => {
canCreateSkillSpy.mockRestore();
}
});
it('resolves model-spec skill names through deployment-aware skill methods', async () => {
const deploymentSkillId = new mongoose.Types.ObjectId();
await AclEntry.create({
principalType: PrincipalType.USER,
principalId: testUser._id,
principalModel: PrincipalModel.USER,
resourceType: ResourceType.SKILL,
resourceId: new mongoose.Types.ObjectId(),
permBits: PermissionBits.VIEW,
grantedBy: testUser._id,
});
const endpointOption = makeEndpointOption();
endpointOption.spec = 'spec-deployment-skill';
endpointOption.agent = Promise.resolve({
id: Constants.EPHEMERAL_AGENT_ID,
name: 'Ephemeral Primary',
provider: 'openai',
model: 'gpt-4',
tools: [],
});
mockInitializeAgent.mockResolvedValue(makePrimaryConfig([]));
const req = makeReq();
req.config.endpoints.agents = { capabilities: ['skills'] };
req.config.modelSpecs = {
list: [{ name: 'spec-deployment-skill', skills: ['deployment-skill'] }],
};
const getSkillByNameSpy = jest.spyOn(getSkillDbMethods(), 'getSkillByName').mockResolvedValue({
_id: deploymentSkillId,
name: 'deployment-skill',
source: 'deployment',
});
const canCreateSkillSpy = jest
.spyOn(getSkillToolDeps(), 'canCreateSkill')
.mockResolvedValue(false);
try {
await initializeClient({
req,
res: {},
signal: new AbortController().signal,
endpointOption,
});
expect(getSkillByNameSpy).toHaveBeenCalledWith(
'deployment-skill',
expect.any(Array),
expect.any(Object),
);
const initializeParams = mockInitializeAgent.mock.calls[0][0];
expect(initializeParams.agent.skills_enabled).toBe(true);
expect(initializeParams.agent.skills).toEqual([deploymentSkillId.toString()]);
} finally {
getSkillByNameSpy.mockRestore();
canCreateSkillSpy.mockRestore();
}
});
});
describe('initializeClient — subagent loading', () => {

View file

@ -7,10 +7,10 @@ import type { TSkillSummary } from 'librechat-data-provider';
import type { MentionOption } from '~/common';
import useInitPopoverInput from '~/hooks/Input/useInitPopoverInput';
import { useLocalize, useSkillActiveState } from '~/hooks';
import { useAgentsMapContext } from '~/Providers';
import { useSkillsInfiniteQuery } from '~/data-provider';
import { isEphemeralAgent } from '~/common';
import { useAgentsMapContext } from '~/Providers';
import { ephemeralAgentByConvoId } from '~/store';
import { isEphemeralAgent } from '~/common';
import { removeCharIfLast } from '~/utils';
import MentionItem from './MentionItem';
import store from '~/store';
@ -50,7 +50,7 @@ export function filterSkillsForPopover(
skills: TSkillSummary[],
ctx: {
agentSkillIds: string[] | null | undefined;
isActive: (skill: Pick<TSkillSummary, '_id' | 'author'>) => boolean;
isActive: (skill: Pick<TSkillSummary, '_id' | 'author' | 'source'>) => boolean;
},
): TSkillSummary[] {
const { agentSkillIds, isActive } = ctx;
@ -128,21 +128,10 @@ function SkillsCommandContent({
const { data, isLoading, isError, fetchNextPage, hasNextPage, isFetchingNextPage } =
useSkillsInfiniteQuery({ limit: 50 });
/* Sticky circuit breaker: once any page request fails, stop auto-fetching
for the lifetime of the popover so a transient API error does not turn
into an unbounded retry loop (isError can flip back to false on the
next attempt, which would otherwise re-arm the auto-fetch effect). */
const paginationBlockedRef = useRef(false);
useEffect(() => {
if (isError) {
paginationBlockedRef.current = true;
}
}, [isError]);
/* Auto-fetch all pages so client-side search covers the full catalog,
not just the first page. The skills API is server-side capped. */
useEffect(() => {
if (paginationBlockedRef.current || isError) {
if (isError) {
return;
}
if (hasNextPage && !isFetchingNextPage) {

View file

@ -435,6 +435,36 @@ describe('SkillsCommand', () => {
expect(screen.queryByRole('button', { name: /Brand Guidelines/i })).toBeNull();
expect(screen.getByRole('button', { name: /Style Guide/i })).toBeInTheDocument();
});
it('resumes catalog pagination after an external refetch clears an error', () => {
const fetchNextPage = jest.fn();
mockUseSkillsInfiniteQuery.mockReturnValue({
data: skillsResponse,
isLoading: false,
isError: true,
fetchNextPage,
hasNextPage: true,
isFetchingNextPage: false,
});
const textAreaRef = makeTextarea('$');
const { rerender } = render(
<SkillsCommand index={0} textAreaRef={textAreaRef} conversationId={CONVO_ID} />,
);
expect(fetchNextPage).not.toHaveBeenCalled();
mockUseSkillsInfiniteQuery.mockReturnValue({
data: skillsResponse,
isLoading: false,
isError: false,
fetchNextPage,
hasNextPage: true,
isFetchingNextPage: false,
});
rerender(<SkillsCommand index={1} textAreaRef={textAreaRef} conversationId={CONVO_ID} />);
expect(fetchNextPage).toHaveBeenCalledTimes(1);
});
});
describe('filterSkillsForPopover', () => {

View file

@ -1,4 +1,4 @@
import { useMemo, useState, useCallback } from 'react';
import { useMemo, useState, useCallback, useEffect } from 'react';
import { Plus, Search } from 'lucide-react';
import { useFormContext, useWatch } from 'react-hook-form';
import { PermissionTypes, Permissions } from 'librechat-data-provider';
@ -11,7 +11,7 @@ import {
OGDialogContent,
OGDialogDescription,
} from '@librechat/client';
import type { TSkill } from 'librechat-data-provider';
import type { TSkill, TSkillSummary } from 'librechat-data-provider';
import type { TranslationKeys } from '~/hooks/useLocalize';
import type { CategoryOption } from './CategoryFilter';
import type { AgentItem } from './items/types';
@ -19,8 +19,8 @@ import type { AgentForm } from '~/common';
import { useLocalize, useHasAccess, useAuthContext, useToolFavorites } from '~/hooks';
import { CreateSkillDialog } from '~/components/Skills/dialogs';
import { skillsEnabledTransition } from './items/mutations';
import { useSkillsInfiniteQuery } from '~/data-provider';
import MarketplaceCatalog from './MarketplaceCatalog';
import { useListSkillsQuery } from '~/data-provider';
import { CategoryIcon } from '~/components/Prompts';
import { buildSkillItems } from './items/catalog';
import ItemDialog from './ItemDialog/ItemDialog';
@ -55,12 +55,30 @@ export default function SkillsDialog({ open, onOpenChange, agentId }: SkillsDial
permissionType: PermissionTypes.SKILLS,
permission: Permissions.CREATE,
});
const { data: skillsData, isLoading: isLoadingSkills } = useListSkillsQuery(
{ limit: 100 },
{ enabled: hasSkillsAccess },
);
const {
data: skillsData,
isLoading: isLoadingSkills,
isError: isSkillsError,
fetchNextPage,
refetch: refetchSkills,
hasNextPage,
isFetchingNextPage,
} = useSkillsInfiniteQuery({ limit: 100 }, { enabled: hasSkillsAccess });
const { favoriteKeys, toggle: toggleFavorite } = useToolFavorites();
useEffect(() => {
if (isSkillsError) {
return;
}
if (hasNextPage && !isFetchingNextPage) {
fetchNextPage();
}
}, [hasNextPage, isFetchingNextPage, isSkillsError, fetchNextPage]);
const handleRetrySkills = useCallback(() => {
void refetchSkills();
}, [refetchSkills]);
const skillsField = useWatch({ control, name: 'skills' });
const selectedIds = useMemo(
() => new Set(((skillsField ?? []) as string[]).map((id) => itemKey({ kind: 'skill', id }))),
@ -73,10 +91,22 @@ export default function SkillsDialog({ open, onOpenChange, agentId }: SkillsDial
const [createOpen, setCreateOpen] = useState(false);
const [detailItem, setDetailItem] = useState<AgentItem | null>(null);
const catalog = useMemo(
() => buildSkillItems(skillsData?.skills ?? [], user?.id),
[skillsData, user?.id],
);
const skills = useMemo(() => {
const allSkills: TSkillSummary[] = [];
const seen = new Set<string>();
for (const page of skillsData?.pages ?? []) {
for (const skill of page.skills) {
if (seen.has(skill._id)) {
continue;
}
seen.add(skill._id);
allSkills.push(skill);
}
}
return allSkills;
}, [skillsData?.pages]);
const catalog = useMemo(() => buildSkillItems(skills, user?.id), [skills, user?.id]);
const categoryOptions = useMemo<CategoryOption[]>(() => {
const seen = new Set<string>();
@ -210,13 +240,24 @@ export default function SkillsDialog({ open, onOpenChange, agentId }: SkillsDial
</div>
<div className="flex-1 overflow-y-auto px-6 py-4">
{isSkillsError && (
<div
role="alert"
className="mb-3 flex items-center justify-between gap-3 rounded-xl border border-border-medium px-3 py-2 text-sm text-text-secondary"
>
<span>{localize('com_ui_skills_load_error')}</span>
<Button type="button" variant="outline" onClick={handleRetrySkills}>
{localize('com_ui_retry')}
</Button>
</div>
)}
<MarketplaceCatalog
items={filtered}
selectedIds={selectedIds}
onToggle={handleToggle}
onConfigure={setDetailItem}
view={view}
isLoadingSkills={isLoadingSkills}
isLoadingSkills={isLoadingSkills || isFetchingNextPage}
skillsInView={view !== 'mine'}
favoriteKeys={favoriteKeys}
onToggleFavorite={toggleFavorite}

View file

@ -0,0 +1,249 @@
import React from 'react';
import '@testing-library/jest-dom/extend-expect';
import { fireEvent, render, screen } from '@testing-library/react';
import type { TSkillSummary } from 'librechat-data-provider';
import SkillsDialog from '../SkillsDialog';
const mockFetchNextPage = jest.fn();
const mockRefetch = jest.fn();
const mockUseListSkillsQuery = jest.fn();
const mockUseSkillsInfiniteQuery = jest.fn();
let mockInfiniteResult: {
data?: { pages: Array<{ skills: TSkillSummary[] }> };
isLoading: boolean;
isError: boolean;
fetchNextPage: jest.Mock;
refetch: jest.Mock;
hasNextPage: boolean;
isFetchingNextPage: boolean;
};
jest.mock('react-hook-form', () => ({
useFormContext: () => ({
control: {},
getValues: (name: string) => (name === 'skills' ? [] : false),
setValue: jest.fn(),
}),
useWatch: () => [],
}));
jest.mock('~/hooks', () => ({
useLocalize: () => (key: string) => {
const values: Record<string, string> = {
com_ui_skills: 'Skills',
com_ui_search_skills: 'Search skills...',
com_ui_skills_dialog_description: 'Browse and add skills to your agent.',
com_ui_skills_load_error: 'Failed to load skills',
com_ui_retry: 'Retry',
};
return values[key] ?? key;
},
useHasAccess: () => true,
useAuthContext: () => ({ user: { id: 'user-1' } }),
useToolFavorites: () => ({
favoriteKeys: new Set<string>(),
toggle: jest.fn(),
}),
}));
jest.mock('~/data-provider', () => ({
useListSkillsQuery: (...args: unknown[]) => mockUseListSkillsQuery(...args),
useSkillsInfiniteQuery: (...args: unknown[]) => mockUseSkillsInfiniteQuery(...args),
}));
jest.mock('~/components/Prompts', () => ({
CategoryIcon: () => <span aria-hidden="true" />,
}));
jest.mock('~/components/Skills/dialogs', () => ({
CreateSkillDialog: () => null,
}));
jest.mock('../CategoryFilter', () => ({
__esModule: true,
default: () => null,
}));
jest.mock('../ItemDialog/ItemDialog', () => ({
__esModule: true,
default: () => null,
}));
jest.mock('../MarketplaceCatalog', () => ({
__esModule: true,
default: ({
items,
ariaLabel,
}: {
items: Array<{ id: string; name: string }>;
ariaLabel: string;
}) => (
<ul aria-label={ariaLabel}>
{items.map((item) => (
<li key={item.id}>{item.name}</li>
))}
</ul>
),
}));
jest.mock('@librechat/client', () => ({
Button: ({ children, ...props }: React.ComponentProps<'button'>) => (
<button type="button" {...props}>
{children}
</button>
),
Input: (props: React.ComponentProps<'input'>) => <input {...props} />,
Radio: () => null,
OGDialog: ({ open, children }: { open: boolean; children: React.ReactNode }) =>
open ? <div>{children}</div> : null,
OGDialogContent: ({ children }: { children: React.ReactNode }) => (
<div role="dialog">{children}</div>
),
OGDialogTitle: ({ children }: { children: React.ReactNode }) => <h2>{children}</h2>,
OGDialogDescription: ({ children }: { children: React.ReactNode }) => <p>{children}</p>,
}));
function makeSkill(id: string, name: string): TSkillSummary {
return {
_id: id,
name,
description: `${name} description`,
author: 'user-1',
authorName: 'E2E User',
version: 1,
source: 'inline',
fileCount: 0,
createdAt: '2026-07-24T00:00:00.000Z',
updatedAt: '2026-07-24T00:00:00.000Z',
};
}
beforeEach(() => {
mockInfiniteResult = {
data: { pages: [{ skills: [makeSkill('first', 'first-skill')] }] },
isLoading: false,
isError: false,
fetchNextPage: mockFetchNextPage,
refetch: mockRefetch,
hasNextPage: true,
isFetchingNextPage: false,
};
mockFetchNextPage.mockReset();
mockRefetch.mockReset();
mockUseListSkillsQuery.mockReset();
mockUseListSkillsQuery.mockReturnValue({
data: { skills: [makeSkill('first', 'first-skill')] },
isLoading: false,
});
mockUseSkillsInfiniteQuery.mockReset();
mockUseSkillsInfiniteQuery.mockImplementation(() => mockInfiniteResult);
});
describe('SkillsDialog pagination', () => {
test('auto-fetches every page and stops after the final page', () => {
const { rerender } = render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(mockUseSkillsInfiniteQuery).toHaveBeenCalledWith({ limit: 100 }, { enabled: true });
expect(mockFetchNextPage).toHaveBeenCalledTimes(1);
mockInfiniteResult = {
...mockInfiniteResult,
isFetchingNextPage: true,
};
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(mockFetchNextPage).toHaveBeenCalledTimes(1);
mockInfiniteResult = {
...mockInfiniteResult,
data: {
pages: [
{ skills: [makeSkill('first', 'first-skill')] },
{ skills: [makeSkill('second', 'second-skill')] },
],
},
hasNextPage: true,
isFetchingNextPage: false,
};
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(mockFetchNextPage).toHaveBeenCalledTimes(2);
expect(screen.getByText('second-skill')).toBeInTheDocument();
mockInfiniteResult = {
...mockInfiniteResult,
isFetchingNextPage: true,
};
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(mockFetchNextPage).toHaveBeenCalledTimes(2);
mockInfiniteResult = {
...mockInfiniteResult,
data: {
pages: [
{ skills: [makeSkill('first', 'first-skill')] },
{ skills: [makeSkill('second', 'second-skill')] },
{ skills: [makeSkill('third', 'third-skill')] },
],
},
hasNextPage: false,
isFetchingNextPage: false,
};
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(screen.getByText('third-skill')).toBeInTheDocument();
expect(mockFetchNextPage).toHaveBeenCalledTimes(2);
});
test('flattens and de-duplicates every loaded page for discovery', () => {
mockInfiniteResult = {
...mockInfiniteResult,
data: {
pages: [
{ skills: [makeSkill('first', 'first-skill')] },
{
skills: [makeSkill('first', 'first-skill'), makeSkill('off-page', 'off-page-skill')],
},
],
},
hasNextPage: false,
};
render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(screen.getAllByText('first-skill')).toHaveLength(1);
expect(screen.getByText('off-page-skill')).toBeInTheDocument();
});
test('shows a recoverable pagination error and retries on request', () => {
mockInfiniteResult = {
...mockInfiniteResult,
isError: true,
};
render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(mockFetchNextPage).not.toHaveBeenCalled();
expect(screen.getByRole('alert')).toHaveTextContent('Failed to load skills');
fireEvent.click(screen.getByRole('button', { name: 'Retry' }));
expect(mockRefetch).toHaveBeenCalledTimes(1);
});
test('resumes pagination after an external refetch clears the error', () => {
mockInfiniteResult = {
...mockInfiniteResult,
isError: true,
};
const { rerender } = render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(mockFetchNextPage).not.toHaveBeenCalled();
mockInfiniteResult = {
...mockInfiniteResult,
isError: false,
};
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
expect(mockFetchNextPage).toHaveBeenCalledTimes(1);
expect(screen.queryByRole('alert')).not.toBeInTheDocument();
});
});

View file

@ -13,6 +13,7 @@ import {
SKILL_DESCRIPTION_MAX_LENGTH,
} from 'librechat-data-provider';
import type { TSkill } from 'librechat-data-provider';
import type { FormEvent } from 'react';
import { useCreateSkillMutation } from '~/data-provider';
import { useLocalize } from '~/hooks';
import { cn } from '~/utils';
@ -98,13 +99,23 @@ export default function CreateSkillDialog({
reset();
};
const handleFormSubmit = (event: FormEvent<HTMLFormElement>) => {
/**
* This dialog can be portaled from inside Agent Builder's form. React
* events still bubble through the component tree across a portal, so an
* unguarded submit also submits (and prematurely creates) the agent.
*/
event.stopPropagation();
return handleSubmit(onSubmit)(event);
};
const submitDisabled = !isValid || isSubmitting || createSkill.isLoading;
return (
<OGDialog open={isOpen} onOpenChange={setIsOpen}>
<OGDialogContent className="w-11/12 max-w-5xl overflow-hidden" showCloseButton={false}>
<form
onSubmit={handleSubmit(onSubmit)}
onSubmit={handleFormSubmit}
className="flex max-h-[80vh] min-w-0 flex-col gap-3 overflow-hidden p-1 sm:gap-4 sm:p-2"
>
<h2 className="text-lg font-bold text-text-primary">

View file

@ -0,0 +1,111 @@
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import type { FormEvent, ReactNode } from 'react';
import CreateSkillDialog from '../CreateSkillDialog';
const mockMutate = jest.fn();
const mockNavigate = jest.fn();
const mockSetIsOpen = jest.fn();
const mockShowToast = jest.fn();
jest.mock('react-router-dom', () => ({
...jest.requireActual('react-router-dom'),
useNavigate: () => mockNavigate,
}));
jest.mock(
'@librechat/client',
() => {
const React = jest.requireActual<typeof import('react')>('react');
const ReactDOM = jest.requireActual<typeof import('react-dom')>('react-dom');
return {
Button: ({
children,
variant: _variant,
...props
}: React.ButtonHTMLAttributes<HTMLButtonElement> & { variant?: string }) =>
React.createElement('button', props, children),
OGDialog: ({ open, children }: { open: boolean; children: ReactNode }) =>
open ? React.createElement(React.Fragment, null, children) : null,
OGDialogContent: ({ children }: { children: ReactNode }) =>
ReactDOM.createPortal(React.createElement('div', null, children), globalThis.document.body),
TextareaAutosize: React.forwardRef<
HTMLTextAreaElement,
React.TextareaHTMLAttributes<HTMLTextAreaElement> & {
minRows?: number;
maxRows?: number;
}
>(({ minRows: _minRows, maxRows: _maxRows, ...props }, ref) =>
React.createElement('textarea', { ...props, ref }),
),
useToastContext: () => ({
showToast: mockShowToast,
}),
};
},
{ virtual: true },
);
jest.mock('~/data-provider', () => ({
useCreateSkillMutation: () => ({
mutate: mockMutate,
isLoading: false,
}),
}));
jest.mock('~/hooks', () => ({
useLocalize: () => (key: string) => {
const translations: Record<string, string> = {
com_ui_name: 'Name',
com_ui_description: 'Description',
com_ui_skill_instructions: 'Instructions',
com_ui_cancel: 'Cancel',
com_ui_create: 'Create',
};
return translations[key] ?? key;
},
}));
jest.mock('~/utils', () => ({
cn: (...classes: Array<string | false | null | undefined>) => classes.filter(Boolean).join(' '),
}));
describe('CreateSkillDialog', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('does not submit an ancestor form when rendered through a portal', async () => {
const handleAgentSubmit = jest.fn((event: FormEvent<HTMLFormElement>) =>
event.preventDefault(),
);
render(
<form onSubmit={handleAgentSubmit}>
<CreateSkillDialog isOpen={true} setIsOpen={mockSetIsOpen} />
</form>,
);
fireEvent.change(screen.getByLabelText('Name'), {
target: { value: 'portal-skill' },
});
fireEvent.change(screen.getByLabelText('Description'), {
target: { value: 'A skill created from Agent Builder' },
});
fireEvent.change(screen.getByLabelText('Instructions'), {
target: { value: 'Apply the portal skill.' },
});
const createButton = screen.getByRole('button', { name: 'Create' });
await waitFor(() => expect(createButton).toBeEnabled());
fireEvent.click(createButton);
await waitFor(() =>
expect(mockMutate).toHaveBeenCalledWith({
name: 'portal-skill',
description: 'A skill created from Agent Builder',
body: 'Apply the portal skill.',
}),
);
expect(handleAgentSubmit).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,142 @@
import React from 'react';
import { QueryKeys } from 'librechat-data-provider';
import { act, renderHook, waitFor } from '@testing-library/react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import type { TSkill, TSkillListResponse } from 'librechat-data-provider';
import type { ReactNode } from 'react';
import { useCreateSkillMutation } from '../mutations';
import { useSkillsInfiniteQuery } from '../queries';
const mockCreateSkill = jest.fn();
const mockListSkills = jest.fn();
jest.mock('librechat-data-provider', () => {
const actual = jest.requireActual('librechat-data-provider');
return {
...actual,
dataService: {
...actual.dataService,
createSkill: (...args: unknown[]) => mockCreateSkill(...args),
listSkills: (...args: unknown[]) => mockListSkills(...args),
},
};
});
function makeSkill(id: string, name: string): TSkill {
return {
_id: id,
name,
description: `${name} description`,
body: `# ${name}`,
author: 'user-1',
authorName: 'E2E User',
version: 1,
source: 'inline',
fileCount: 0,
createdAt: '2026-07-24T00:00:00.000Z',
updatedAt: '2026-07-24T00:00:00.000Z',
} as TSkill;
}
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>((resolvePromise) => {
resolve = resolvePromise;
});
return { promise, resolve };
}
describe('skill creation cache updates', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('keeps a created skill when a stale next-page request finishes afterward', async () => {
const queryClient = new QueryClient({
defaultOptions: {
queries: { retry: false },
mutations: { retry: false },
},
});
const firstSkill = makeSkill('first-id', 'first-skill');
const createdSkill = makeSkill('created-id', 'created-skill');
const stalePage = deferred<TSkillListResponse>();
const firstPage: TSkillListResponse = {
skills: [firstSkill],
has_more: true,
after: 'cursor-1',
};
const freshPage: TSkillListResponse = {
skills: [createdSkill, firstSkill],
has_more: false,
after: null,
};
const queryKey = [QueryKeys.skills, 'infinite', '', '', 100];
queryClient.setQueryData(queryKey, {
pages: [firstPage],
pageParams: [undefined],
});
mockCreateSkill.mockResolvedValue(createdSkill);
mockListSkills.mockImplementation(({ cursor }: { cursor?: string }) =>
cursor === 'cursor-1' ? stalePage.promise : Promise.resolve(freshPage),
);
const cancelQueries = jest.spyOn(queryClient, 'cancelQueries');
const wrapper = ({ children }: { children: ReactNode }) => (
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
);
const { result } = renderHook(
() => ({
list: useSkillsInfiniteQuery({ limit: 100 }),
create: useCreateSkillMutation(),
}),
{ wrapper },
);
let staleRequest: Promise<unknown>;
act(() => {
staleRequest = result.current.list.fetchNextPage();
});
await waitFor(() =>
expect(mockListSkills).toHaveBeenCalledWith(
expect.objectContaining({ cursor: 'cursor-1', limit: 100 }),
),
);
await act(async () => {
await result.current.create.mutateAsync({
name: createdSkill.name,
description: createdSkill.description,
body: createdSkill.body,
});
});
expect(cancelQueries).toHaveBeenCalledWith([QueryKeys.skills]);
await waitFor(() =>
expect(
queryClient
.getQueryData<{ pages: TSkillListResponse[] }>(queryKey)
?.pages.flatMap((page) => page.skills)
.map((skill) => skill._id),
).toContain(createdSkill._id),
);
stalePage.resolve({
skills: [makeSkill('second-id', 'second-skill')],
has_more: false,
after: null,
});
await act(async () => {
await Promise.allSettled([staleRequest]);
});
expect(
queryClient
.getQueryData<{ pages: TSkillListResponse[] }>(queryKey)
?.pages.flatMap((page) => page.skills)
.map((skill) => skill._id),
).toContain(createdSkill._id);
queryClient.clear();
});
});

View file

@ -1,6 +1,5 @@
import { useMutation, useQueryClient } from '@tanstack/react-query';
import { QueryKeys, dataService } from 'librechat-data-provider';
import type { InfiniteData, QueryKey, UseMutationResult } from '@tanstack/react-query';
import { useMutation, useQueryClient } from '@tanstack/react-query';
import type {
TSkill,
TSkillFile,
@ -21,6 +20,7 @@ import type {
ImportSkillOptions,
DeleteSkillFileOptions,
} from 'librechat-data-provider';
import type { InfiniteData, QueryKey, UseMutationResult } from '@tanstack/react-query';
function isInfiniteSkillData(
data: TSkillListResponse | InfiniteData<TSkillListResponse>,
@ -49,14 +49,21 @@ function addSkillToCachedLists(
queryClient.setQueriesData<TSkillCacheEntry>([QueryKeys.skills], (data) => {
if (!data) return data;
if (isInfiniteSkillData(data)) {
const pages = data.pages.map((page) => ({
...page,
skills: page.skills.filter((existing) => existing._id !== skill._id),
}));
return {
...data,
pages: data.pages.map((page, i) =>
pages: pages.map((page, i) =>
i === 0 ? { ...page, skills: [skill, ...page.skills] } : page,
),
};
}
return { ...data, skills: [skill, ...data.skills] };
return {
...data,
skills: [skill, ...data.skills.filter((existing) => existing._id !== skill._id)],
};
});
}
@ -119,9 +126,16 @@ export const useCreateSkillMutation = (
return useMutation({
mutationFn: (payload: TCreateSkill) => dataService.createSkill(payload),
...rest,
onSuccess: (skill, variables, context) => {
onSuccess: async (skill, variables, context) => {
/**
* An infinite skill query may have captured its pre-create pages before
* this mutation completed. Cancel it before the cache write so a late
* cursor response cannot replace the newly-created row.
*/
await queryClient.cancelQueries([QueryKeys.skills]);
queryClient.setQueryData<TSkill>([QueryKeys.skill, skill._id], skill);
addSkillToCachedLists(queryClient, skill);
void queryClient.invalidateQueries([QueryKeys.skills]);
if (onSuccess) onSuccess(skill, variables, context);
},
});
@ -139,9 +153,11 @@ export const useImportSkillMutation = (
return useMutation({
mutationFn: (formData: FormData) => dataService.importSkill(formData),
...rest,
onSuccess: (skill, variables, context) => {
onSuccess: async (skill, variables, context) => {
await queryClient.cancelQueries([QueryKeys.skills]);
queryClient.setQueryData<TSkill>([QueryKeys.skill, skill._id], skill);
addSkillToCachedLists(queryClient, skill);
void queryClient.invalidateQueries([QueryKeys.skills]);
if (onSuccess) onSuccess(skill, variables, context);
},
});

View file

@ -0,0 +1,25 @@
import { resolveSkillDefaultActive } from '../useSkillActiveState';
describe('resolveSkillDefaultActive', () => {
test('activates deployment skills for every user by default', () => {
expect(
resolveSkillDefaultActive(
{ author: 'deployment-registry', source: 'deployment' },
'user-1',
false,
),
).toBe(true);
});
test('activates owned inline skills by default', () => {
expect(resolveSkillDefaultActive({ author: 'user-1', source: 'inline' }, 'user-1', false)).toBe(
true,
);
});
test('uses the shared-skill configuration for non-owned inline skills', () => {
const skill = { author: 'user-2', source: 'inline' } as const;
expect(resolveSkillDefaultActive(skill, 'user-1', false)).toBe(false);
expect(resolveSkillDefaultActive(skill, 'user-1', true)).toBe(true);
});
});

View file

@ -1,8 +1,8 @@
import { useCallback, useEffect, useMemo } from 'react';
import { useQueryClient } from '@tanstack/react-query';
import { QueryKeys } from 'librechat-data-provider';
import { useToastContext } from '@librechat/client';
import type { TSkillStatesResponse } from 'librechat-data-provider';
import { useQueryClient } from '@tanstack/react-query';
import type { TSkillStatesResponse, TSkillSummary } from 'librechat-data-provider';
import {
useGetSkillStatesQuery,
useUpdateSkillStatesMutation,
@ -12,6 +12,7 @@ import { useAuthContext, useLocalize } from '~/hooks';
import { logger } from '~/utils';
const EMPTY_STATES: TSkillStatesResponse = {};
type SkillActiveTarget = Pick<TSkillSummary, '_id' | 'author' | 'source'>;
interface WriteQueue {
pending: TSkillStatesResponse | null;
@ -65,19 +66,28 @@ function getWriteQueue(userId: string): WriteQueue {
/**
* Resolves the default active state for a skill the user has never toggled.
*
* - Deployment skills default to **active** for every user.
* - Owned skills (author === currentUser) default to **active**.
* - Shared skills default to the `defaultActiveOnShare` config value (default `false`).
*/
function resolveDefault(author: string, userId: string, defaultActiveOnShare: boolean): boolean {
return author === userId ? true : defaultActiveOnShare;
export function resolveSkillDefaultActive(
skill: Pick<SkillActiveTarget, 'author' | 'source'>,
userId: string,
defaultActiveOnShare: boolean,
): boolean {
if (skill.source === 'deployment' || skill.author === userId) {
return true;
}
return defaultActiveOnShare;
}
/**
* Hook for managing per-user skill active/inactive state.
*
* The `skillStates` map stores explicit overrides (`{ [skillId]: boolean }`).
* Skills absent from the map use the ownership-based default: owned -> active,
* shared -> `defaultActiveOnShare` from the interface config. Toggles that
* Skills absent from the map use the source/ownership default: deployment and
* owned skills -> active, shared skills -> `defaultActiveOnShare` from the
* interface config. Toggles that
* land on the resolved default remove the key from the map rather than
* persisting a redundant entry, keeping `skillStates` strictly an exceptions
* list (otherwise rapid round-trip toggles would exhaust the 200-entry cap).
@ -165,18 +175,18 @@ export default function useSkillActiveState() {
}, [userId, updateMutation, showToast, localize]);
const isActive = useCallback(
(skill: { _id: string; author: string }): boolean => {
(skill: SkillActiveTarget): boolean => {
const override = skillStates[skill._id];
if (override !== undefined) {
return override;
}
return resolveDefault(skill.author, userId, defaultActiveOnShare);
return resolveSkillDefaultActive(skill, userId, defaultActiveOnShare);
},
[skillStates, userId, defaultActiveOnShare],
);
const toggle = useCallback(
(skill: { _id: string; author: string }) => {
(skill: SkillActiveTarget) => {
if (!canToggle || !userId) {
return;
}
@ -184,7 +194,7 @@ export default function useSkillActiveState() {
const cached =
queryClient.getQueryData<TSkillStatesResponse>([QueryKeys.skillStates]) ?? EMPTY_STATES;
const baseline = queue.pending ?? cached;
const defaultValue = resolveDefault(skill.author, userId, defaultActiveOnShare);
const defaultValue = resolveSkillDefaultActive(skill, userId, defaultActiveOnShare);
const override = baseline[skill._id];
const currentActive = override !== undefined ? override : defaultValue;
const nextValue = !currentActive;

View file

@ -145,6 +145,7 @@ modelSpecs:
model: 'mock-model-a'
skills:
- 'e2e-model-spec-allowed'
- 'e2e-deployment-skill'
- 'e2e-model-spec-missing'
- 'e2e-model-spec-inaccessible'

View file

@ -18,7 +18,9 @@ const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;
const CREATE_SKILL_MARKER = 'E2E_CREATE_SKILL:';
const EDIT_SKILL_MARKER = 'E2E_EDIT_SKILL:';
const ASSERT_MODEL_SPEC_SKILLS_MARKER = 'E2E_ASSERT_MODEL_SPEC_SKILLS';
const ASSERT_SKILLS_MARKER = 'E2E_ASSERT_SKILLS:';
const ASSERT_MANUAL_SKILL_MARKER = 'E2E_ASSERT_MANUAL_SKILL:';
const INVOKE_SKILL_MARKER = 'E2E_INVOKE_SKILL:';
const ASSERT_PROVIDER_FILE_MARKER = 'E2E_ASSERT_PROVIDER_FILE:';
const ASSERT_AGENT_CONTEXT_MARKER = 'E2E_ASSERT_AGENT_CONTEXT:';
const ASSERT_QUOTE_MARKER = 'E2E_ASSERT_QUOTE:';
@ -34,7 +36,9 @@ const BACKGROUND_DISPATCH_MARKER = 'E2E_BACKGROUND_DISPATCH:';
const BACKGROUND_COLLECT_MARKER = 'E2E_BACKGROUND_COLLECT:';
const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
const MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed';
const SKILL_ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
const MANUAL_SKILL_ASSERTION_FINAL_TEXT = 'E2E manual skill assertion passed';
const SKILL_TOOL_ASSERTION_FINAL_TEXT = 'E2E skill tool assertion passed';
const PROVIDER_FILE_ASSERTION_FINAL_TEXT = 'E2E provider file assertion passed';
const AGENT_CONTEXT_ASSERTION_FINAL_TEXT = 'E2E agent context assertion passed';
const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed';
@ -55,9 +59,9 @@ const CHECK_BACKGROUND_TASK_TOOL_NAME = 'check_background_task';
const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch';
const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect';
const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed';
const MODEL_SPEC_MISSING_SKILL = 'e2e-model-spec-missing';
const MODEL_SPEC_INACCESSIBLE_SKILL = 'e2e-model-spec-inaccessible';
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
const ALWAYS_APPLY_BODY_MARKER = 'E2E_ALWAYS_APPLY_BODY_MARKER';
const DEPLOYMENT_SKILL_BODY_MARKER = 'E2E deployment skill loaded through Playwright';
const SKILL_DESCRIPTION =
'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.';
const EDITED_SKILL_DESCRIPTION =
@ -165,13 +169,30 @@ function collectToolNames(agents) {
return names;
}
function collectAdditionalInstructions(agents) {
return (agents ?? [])
.map((agent) =>
typeof agent?.additional_instructions === 'string' ? agent.additional_instructions : '',
)
.filter(Boolean)
.join('\n');
async function getStreamAgentView({ graph, messages, options, runManager }) {
let agentId = runManager?.metadata?.agentId ?? options?.metadata?.agentId;
let agentContext;
if (typeof agentId === 'string') {
agentContext = graph?.agentContexts?.get(agentId);
} else if (graph?.agentContexts?.size === 1) {
[agentId, agentContext] = graph.agentContexts.entries().next().value;
}
/**
* Graph.attemptInvoke intentionally sends test override models the pruned
* messages directly, bypassing the production model's systemRunnable pipe.
* Apply that agent's runnable here so assertions inspect the same complete
* prompt (system catalog plus messages) that a real provider receives.
*/
const systemRunnable = agentContext?.systemRunnable;
const promptMessages =
systemRunnable && typeof systemRunnable.invoke === 'function'
? await systemRunnable.invoke(messages)
: messages;
return {
agentId,
messages: promptMessages,
toolNames: collectToolNames(agentContext ? [agentContext] : []),
};
}
function collectPromptText(value, parts = []) {
@ -443,7 +464,7 @@ class UsageEmittingFakeChatModel extends FakeChatModel {
async *_streamResponseChunks(messages, options, runManager) {
let outputChars = 0;
const dynamicResponse = this.resolveOnStream?.(messages);
const dynamicResponse = await this.resolveOnStream?.(messages, options, runManager);
const chunkStream = dynamicResponse
? this.streamDynamicResponseChunks({
responses: dynamicResponse.responses,
@ -502,45 +523,208 @@ function overrideModel({ graph, responses, sleep, toolCalls, thrownError, resolv
});
}
function modelSpecSkillAssertionResponses({ agents, messages, toolNames }) {
const failures = [];
const additionalInstructions = collectAdditionalInstructions(agents);
const skillPrimeMessages = collectSkillPrimeMessages(messages);
const alwaysApplyPrime = skillPrimeMessages.find(
(message) => message.name === MODEL_SPEC_ACCESSIBLE_SKILL && message.trigger === 'always-apply',
);
function parseSkillAssertion(text, agentId) {
const markerValue = getMarkerValue(text, ASSERT_SKILLS_MARKER);
const sections = markerValue
.split(';')
.map((section) => section.trim())
.filter(Boolean);
const isAgentScoped = sections.some((section) => section.includes('='));
let entriesValue = markerValue;
if (isAgentScoped) {
// Parallel agents receive a per-run `____N` suffix while the request and
// persisted Agent Builder state retain the stable agent id.
const persistedAgentId =
typeof agentId === 'string' ? agentId.replace(/____\d+$/, '') : agentId;
const prefixes = [`${agentId}=`, `${persistedAgentId}=`];
const scopedSection = sections.find((section) =>
prefixes.some((prefix) => section.startsWith(prefix)),
);
if (!scopedSection) {
return {
required: [],
requiredBodies: [],
forbidden: [],
error: `no skill assertion was configured for agent ${agentId ?? 'unknown'}`,
};
}
const prefix = prefixes.find((candidate) => scopedSection.startsWith(candidate));
entriesValue = scopedSection.slice(prefix.length);
}
if (!toolNames.has(SKILL_TOOL_NAME)) {
const entries = entriesValue
.split(',')
.map((entry) => entry.trim())
.filter(Boolean);
return entries.reduce(
(assertion, entry) => {
if (entry.startsWith('!')) {
const name = entry.slice(1);
if (name) {
assertion.forbidden.push(name);
}
return assertion;
}
if (entry.startsWith('*')) {
const name = entry.slice(1);
if (name) {
assertion.required.push(name);
assertion.requiredBodies.push(name);
}
return assertion;
}
assertion.required.push(entry);
return assertion;
},
{ required: [], requiredBodies: [], forbidden: [], error: undefined },
);
}
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function promptHasSkillCatalogEntry(promptText, skillName) {
if (!promptText.includes('## Available Skills')) {
return false;
}
return new RegExp(`(?:^|\\n)- ${escapeRegExp(skillName)}(?::|\\s*(?:\\n|$))`, 'm').test(
promptText,
);
}
function expectedSkillBodyMarker(skillName) {
if (skillName === MODEL_SPEC_ACCESSIBLE_SKILL) {
return ALWAYS_APPLY_BODY_MARKER;
}
if (skillName === DEPLOYMENT_SKILL_NAME) {
return DEPLOYMENT_SKILL_BODY_MARKER;
}
return `# ${skillName}`;
}
function skillAssertionResponses({ messages, assertion, toolNames }) {
const failures = [];
if (assertion.error) {
failures.push(assertion.error);
}
const promptText = collectPromptText(messages).join('\n');
const skillPrimeMessages = collectSkillPrimeMessages(messages);
if (assertion.required.length > 0 && !toolNames.has(SKILL_TOOL_NAME)) {
failures.push(`${SKILL_TOOL_NAME} tool was not advertised`);
}
if (!additionalInstructions.includes(MODEL_SPEC_ACCESSIBLE_SKILL)) {
failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} was not present in the model-visible catalog`);
if (assertion.required.length === 0 && toolNames.has(SKILL_TOOL_NAME)) {
failures.push(`${SKILL_TOOL_NAME} tool was unexpectedly advertised`);
}
if (additionalInstructions.includes(MODEL_SPEC_MISSING_SKILL)) {
failures.push(`${MODEL_SPEC_MISSING_SKILL} leaked into the model-visible catalog`);
for (const name of assertion.required) {
if (!promptHasSkillCatalogEntry(promptText, name)) {
failures.push(`${name} was not present in the model-visible catalog`);
}
}
if (additionalInstructions.includes(MODEL_SPEC_INACCESSIBLE_SKILL)) {
failures.push(`${MODEL_SPEC_INACCESSIBLE_SKILL} leaked into the model-visible catalog`);
for (const name of assertion.requiredBodies) {
const expectedMarker = expectedSkillBodyMarker(name);
const taggedBody = skillPrimeMessages.find((message) => message.name === name);
if (!taggedBody?.content.includes(expectedMarker)) {
failures.push(`${name} body was missing its expected marker "${expectedMarker}"`);
}
}
if (!alwaysApplyPrime) {
failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} was not always-apply primed`);
} else if (!alwaysApplyPrime.content.includes(ALWAYS_APPLY_BODY_MARKER)) {
failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} always-apply body was missing its marker`);
for (const name of assertion.forbidden) {
if (promptHasSkillCatalogEntry(promptText, name)) {
failures.push(`${name} leaked into the model-visible catalog`);
}
if (skillPrimeMessages.some((message) => message.name === name)) {
failures.push(`${name} was unexpectedly primed`);
}
if (name === MODEL_SPEC_ACCESSIBLE_SKILL && promptText.includes(ALWAYS_APPLY_BODY_MARKER)) {
failures.push(`${name} always-apply body marker leaked into the model prompt`);
}
if (name === DEPLOYMENT_SKILL_NAME && promptText.includes(DEPLOYMENT_SKILL_BODY_MARKER)) {
failures.push(`${name} always-apply body marker leaked into the model prompt`);
}
}
if (skillPrimeMessages.some((message) => message.name === MODEL_SPEC_MISSING_SKILL)) {
failures.push(`${MODEL_SPEC_MISSING_SKILL} was unexpectedly primed`);
}
if (skillPrimeMessages.some((message) => message.name === MODEL_SPEC_INACCESSIBLE_SKILL)) {
failures.push(`${MODEL_SPEC_INACCESSIBLE_SKILL} was unexpectedly primed`);
}
if (failures.length > 0) {
return {
responses: [`E2E model spec skill assertion failed: ${failures.join('; ')}`],
responses: [`E2E skill assertion failed: ${failures.join('; ')}`],
};
}
return {
responses: [`${MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT}: ${MODEL_SPEC_ACCESSIBLE_SKILL}`],
responses: [
`${SKILL_ASSERTION_FINAL_TEXT}: ${
assertion.required.length > 0 ? assertion.required.join(', ') : 'none'
}`,
],
};
}
function manualSkillAssertionResponses({ messages, skillName }) {
const taggedPrime = collectSkillPrimeMessages(messages).find(
(message) => message.name === skillName && message.trigger === 'manual',
);
if (!taggedPrime) {
return {
responses: [`E2E manual skill assertion failed: ${skillName} was not manually primed`],
};
}
if (!taggedPrime.content.includes(`# ${skillName}`)) {
return {
responses: [`E2E manual skill assertion failed: ${skillName} body was missing`],
};
}
return {
responses: [`${MANUAL_SKILL_ASSERTION_FINAL_TEXT}: ${skillName}`],
};
}
/**
* Exercises the real event-driven `skill` handler. The first model call emits
* the tool request; the second verifies both the visible tool result and the
* body-bearing meta HumanMessage that ToolNode reinjected for the model.
*/
function skillToolInvocationResponses({ skillName, toolNames }) {
if (!toolNames.has(SKILL_TOOL_NAME)) {
return {
responses: [`E2E skill tool assertion failed: ${SKILL_TOOL_NAME} was not advertised`],
};
}
return {
responses: ['', ''],
toolCalls: [
{
id: `call_e2e_skill_${skillName}`,
name: SKILL_TOOL_NAME,
args: { skillName },
type: 'tool_call',
},
],
resolveOnStream: (streamMessages) => {
const toolResult = findLastToolMessageText(
streamMessages,
`Skill "${skillName}" loaded. Follow the instructions below.`,
);
if (!toolResult) {
return null;
}
const expectedMarker = expectedSkillBodyMarker(skillName);
const modelInvokedPrime = collectSkillPrimeMessages(streamMessages).find(
(message) =>
message.name === skillName &&
message.trigger == null &&
message.content.includes(expectedMarker),
);
if (!modelInvokedPrime) {
return {
responses: [
`E2E skill tool assertion failed: ${skillName} body was not reinjected with marker "${expectedMarker}"`,
],
};
}
return {
responses: [`${SKILL_TOOL_ASSERTION_FINAL_TEXT}: ${skillName}`],
};
},
};
}
@ -740,7 +924,7 @@ function backgroundCollectResponses(messages, toolNames) {
};
}
function resolveResponses({ agents, messages, text, toolNames }) {
function resolveResponses({ graph, messages, text, toolNames }) {
const reply = replyResponses(text);
if (reply) {
return reply;
@ -769,8 +953,50 @@ function resolveResponses({ agents, messages, text, toolNames }) {
return quoteAssertion;
}
if (text.includes(ASSERT_MODEL_SPEC_SKILLS_MARKER)) {
return modelSpecSkillAssertionResponses({ agents, messages, toolNames });
if (text.includes(ASSERT_SKILLS_MARKER)) {
return {
responses: [MOCK_REPLY],
resolveOnStream: async (streamMessages, streamOptions, runManager) => {
const agentView = await getStreamAgentView({
graph,
messages: streamMessages,
options: streamOptions,
runManager,
});
return skillAssertionResponses({
messages: agentView.messages,
assertion: parseSkillAssertion(text, agentView.agentId),
toolNames: agentView.toolNames,
});
},
};
}
if (text.includes(ASSERT_MANUAL_SKILL_MARKER)) {
const skillName = getMarkerValue(text, ASSERT_MANUAL_SKILL_MARKER);
return {
responses: [MOCK_REPLY],
resolveOnStream: async (streamMessages, streamOptions, runManager) => {
const agentView = await getStreamAgentView({
graph,
messages: streamMessages,
options: streamOptions,
runManager,
});
return manualSkillAssertionResponses({
messages: agentView.messages,
skillName,
});
},
};
}
const invokedSkillName = getMarkerValue(text, INVOKE_SKILL_MARKER);
if (invokedSkillName) {
return skillToolInvocationResponses({
skillName: invokedSkillName,
toolNames,
});
}
const createSkillName = getRequestedSkillName(text, CREATE_SKILL_MARKER);
@ -824,7 +1050,7 @@ module.exports = function fakeModelHook(run, context) {
const text = getLatestUserText(context?.messages);
const toolNames = collectToolNames(context?.agents);
const { responses, sleep, toolCalls, thrownError, resolveOnStream } = resolveResponses({
agents: context?.agents,
graph,
messages: context?.messages,
text,
toolNames,

View file

@ -0,0 +1,174 @@
import { expect, test } from '@playwright/test';
import type { Page } from '@playwright/test';
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
import { NEW_CHAT_PATH, fetchJson, getAccessToken, requestJson, sendMessage } from './helpers';
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
const ASSERTION_MARKER = 'E2E_ASSERT_SKILLS:';
const ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
type SkillSummary = {
_id: string;
name: string;
source?: string;
};
type AgentResponse = {
id: string;
name?: string | null;
skills?: string[];
skills_enabled?: boolean;
};
type AgentChatPayload = {
addedConvo?: {
agent_id?: string;
endpoint?: string;
};
};
async function getDeploymentSkill(page: Page, token: string): Promise<SkillSummary> {
const result = await fetchJson<{ skills?: SkillSummary[] }>(
page,
`/api/skills?search=${encodeURIComponent(DEPLOYMENT_SKILL_NAME)}&limit=10`,
token,
);
const skill = result.skills?.find((item) => item.name === DEPLOYMENT_SKILL_NAME);
expect(skill).toMatchObject({
name: DEPLOYMENT_SKILL_NAME,
source: 'deployment',
});
return skill!;
}
async function selectAddedAgent(page: Page, agentName: string) {
const messageInput = page.getByRole('textbox', { name: 'Message input' });
await messageInput.click();
await messageInput.pressSequentially('+');
const addedModelSearch = page.getByPlaceholder(
'Add a model or preset for an additional response',
);
await expect(addedModelSearch).toBeVisible();
await addedModelSearch.fill(agentName);
const agentOption = page.locator('button[id^="add-convo-item-"]').filter({ hasText: agentName });
await expect(agentOption).toHaveCount(1);
await agentOption.click();
await expect(page.getByText(`+ ${agentName}`, { exact: true })).toBeVisible();
await expect(messageInput).toHaveValue('');
}
async function selectPrimaryAgent(page: Page, agentName: string) {
const form = await openAgentBuilder(page);
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
await page.getByRole('option', { name: agentName, exact: true }).click();
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
await form.getByRole('button', { name: 'Select Agent' }).click();
}
async function settleCleanup(tasks: Promise<unknown>[]) {
const results = await Promise.allSettled(tasks);
const failures = results.filter(
(result): result is PromiseRejectedResult => result.status === 'rejected',
);
if (failures.length > 0) {
throw failures[0].reason;
}
}
test.describe('added agent skills', () => {
test('keeps an added agent deployment-skill catalog scoped to that agent', async ({ page }) => {
test.setTimeout(120000);
const primaryAgentName = uniqueAgentName('E2E Primary Skill Agent');
const addedAgentName = uniqueAgentName('E2E Added Skill Agent');
let primaryAgentId: string | undefined;
let addedAgentId: string | undefined;
try {
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
const token = await getAccessToken(page);
const deploymentSkill = await getDeploymentSkill(page, token);
const primaryAgent = await requestJson<AgentResponse>(page, {
path: '/api/agents',
token,
method: 'POST',
body: {
name: primaryAgentName,
description: 'Primary agent without skills for added-agent scope coverage.',
instructions: 'Respond as the primary agent.',
provider: 'Mock Provider A',
model: 'mock-model-a',
model_parameters: {},
skills_enabled: false,
skills: [],
},
});
primaryAgentId = primaryAgent.id;
const addedAgent = await requestJson<AgentResponse>(page, {
path: '/api/agents',
token,
method: 'POST',
body: {
name: addedAgentName,
description: 'Persisted added-agent deployment skill regression fixture.',
instructions: 'Use the configured deployment skill when responding.',
provider: 'Mock Provider B',
model: 'mock-model-b',
model_parameters: {},
skills_enabled: true,
skills: [deploymentSkill._id],
},
});
addedAgentId = addedAgent.id;
expect(addedAgent).toMatchObject({
name: addedAgentName,
skills_enabled: true,
skills: [deploymentSkill._id],
});
await selectPrimaryAgent(page, primaryAgentName);
await selectAddedAgent(page, addedAgentName);
const scopedAssertion = `${ASSERTION_MARKER}${primaryAgentId}=!${DEPLOYMENT_SKILL_NAME};${addedAgentId}=${DEPLOYMENT_SKILL_NAME}`;
const response = await sendMessage(
page,
[scopedAssertion, 'Verify the persisted added-agent skill catalog.'].join('\n'),
);
expect(response.ok()).toBeTruthy();
const payload = response.request().postDataJSON() as AgentChatPayload;
expect(payload.addedConvo).toMatchObject({
agent_id: addedAgentId,
endpoint: 'agents',
});
const parallelGroup = page
.getByTestId('messages-view')
.locator('.sibling-content-group')
.last();
const addedAgentColumn = parallelGroup
.locator(':scope > div')
.filter({ hasText: addedAgentName });
await expect(addedAgentColumn).toHaveCount(1);
await expect(addedAgentColumn).toContainText(
`${ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}`,
{ timeout: 30000 },
);
const primaryColumn = parallelGroup
.locator(':scope > div')
.filter({ hasText: primaryAgentName });
await expect(primaryColumn).toHaveCount(1);
await expect(primaryColumn).toContainText(`${ASSERTION_FINAL_TEXT}: none`, {
timeout: 30000,
});
} finally {
await settleCleanup([cleanupAgent(page, primaryAgentId), cleanupAgent(page, addedAgentId)]);
}
});
});

View file

@ -0,0 +1,810 @@
import { expect, test } from '@playwright/test';
import type { Locator, Page, Request, Response } from '@playwright/test';
import type { AgentDetail } from './agents.helpers';
import {
cleanupAgent,
openAgentBuilder,
selectMockModel,
uniqueAgentName,
waitForPersistedAgent,
} from './agents.helpers';
import { fetchJson, getAccessToken, requestJson, sendMessage } from './helpers';
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
const SKILL_ASSERTION_MARKER = 'E2E_ASSERT_SKILLS:';
const SKILL_ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
const MANUAL_SKILL_ASSERTION_MARKER = 'E2E_ASSERT_MANUAL_SKILL:';
const MANUAL_SKILL_ASSERTION_FINAL_TEXT = 'E2E manual skill assertion passed';
const SKILL_TOOL_INVOCATION_MARKER = 'E2E_INVOKE_SKILL:';
const SKILL_TOOL_ASSERTION_FINAL_TEXT = 'E2E skill tool assertion passed';
const SKILL_PICKER_PLACEHOLDER = 'Select a Skill by name';
const NO_SKILLS_FOUND_TEXT = 'No skills found';
const INLINE_SKILL_DESCRIPTION =
'Use this inline skill to verify Agent Builder skill selection and persistence.';
type SkillSummary = {
_id: string;
name: string;
description: string;
source?: string;
sourceMetadata?: {
deployment?: boolean;
};
};
type SkillAgentDetail = AgentDetail & {
skills?: string[];
skills_enabled?: boolean;
};
type AgentSkillPayload = {
name?: string;
skills?: string[];
skills_enabled?: boolean;
manualSkills?: string[];
};
const uniqueSkillName = (prefix: string) =>
`${prefix}-${Date.now().toString(36)}-${Math.floor(Math.random() * 1e4)}`;
async function createInlineSkill(page: Page, token: string, name: string): Promise<SkillSummary> {
return requestJson<SkillSummary>(page, {
path: '/api/skills',
token,
method: 'POST',
body: {
name,
description: INLINE_SKILL_DESCRIPTION,
body: `# ${name}\n\nUse this skill in Agent Builder end-to-end tests.`,
},
});
}
async function createInlineSkills(
page: Page,
token: string,
names: string[],
onCreated?: (skill: SkillSummary) => void,
): Promise<SkillSummary[]> {
const created: SkillSummary[] = [];
const batchSize = 10;
for (let index = 0; index < names.length; index += batchSize) {
const batch = await Promise.allSettled(
names.slice(index, index + batchSize).map((name) => createInlineSkill(page, token, name)),
);
for (const result of batch) {
if (result.status === 'fulfilled') {
created.push(result.value);
onCreated?.(result.value);
}
}
const failed = batch.find((result) => result.status === 'rejected');
if (failed?.status === 'rejected') {
throw failed.reason;
}
}
return created;
}
async function getDeploymentSkill(page: Page, token: string): Promise<SkillSummary> {
const result = await fetchJson<{ skills?: SkillSummary[] }>(
page,
`/api/skills?search=${encodeURIComponent(DEPLOYMENT_SKILL_NAME)}&limit=10`,
token,
);
const skill = result.skills?.find((item) => item.name === DEPLOYMENT_SKILL_NAME);
expect(skill, 'Expected the deployment skill fixture to be accessible').toMatchObject({
name: DEPLOYMENT_SKILL_NAME,
source: 'deployment',
sourceMetadata: { deployment: true },
});
return skill!;
}
async function deleteSkill(page: Page, token: string, skillId: string): Promise<void> {
await requestJson(page, {
path: `/api/skills/${encodeURIComponent(skillId)}`,
token,
method: 'DELETE',
});
}
async function deleteSkills(page: Page, token: string, skillIds: string[]): Promise<void> {
const batchSize = 10;
const failures: unknown[] = [];
for (let index = 0; index < skillIds.length; index += batchSize) {
const results = await Promise.allSettled(
skillIds.slice(index, index + batchSize).map((skillId) => deleteSkill(page, token, skillId)),
);
for (const result of results) {
if (result.status === 'rejected') {
failures.push(result.reason);
}
}
}
if (failures.length > 0) {
throw failures[0];
}
}
async function settleCleanup(tasks: Promise<unknown>[]): Promise<void> {
const results = await Promise.allSettled(tasks);
const failures = results.filter(
(result): result is PromiseRejectedResult => result.status === 'rejected',
);
if (failures.length > 0) {
throw failures[0].reason;
}
}
async function openSkillsDialog(page: Page, form: Locator): Promise<Locator> {
await form.getByRole('button', { name: 'Add Skills', exact: true }).click();
const dialog = page
.getByRole('dialog')
.filter({ hasText: 'Browse and add skills to your agent.' });
await expect(dialog).toBeVisible();
return dialog;
}
async function skillCard(dialog: Locator, name: string): Promise<Locator> {
const item = dialog
.getByRole('list', { name: 'Skills' })
.getByRole('listitem')
.filter({ hasText: name });
await expect(item).toHaveCount(1);
const card = item.getByRole('button').filter({ hasText: name }).first();
await expect(card).toBeVisible();
return card;
}
async function selectSkill(dialog: Locator, name: string): Promise<void> {
const card = await skillCard(dialog, name);
await card.click();
await expect(card).toHaveAttribute('aria-pressed', 'true');
}
async function waitForAgentMutation(
page: Page,
method: 'POST' | 'PATCH',
agentId?: string,
): Promise<Response> {
return page.waitForResponse(
(response) => {
const { pathname } = new URL(response.url());
const expectedPath = agentId ? `/api/agents/${agentId}` : '/api/agents';
return response.request().method() === method && pathname === expectedPath;
},
{ timeout: 30000 },
);
}
async function selectAgent(page: Page, form: Locator, agentName: string): Promise<void> {
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
await page.getByRole('option', { name: agentName, exact: true }).click();
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
}
async function openSkillPicker(page: Page): Promise<Locator> {
const messageInput = page.getByRole('textbox', { name: 'Message input' });
await messageInput.click();
await messageInput.fill('');
await messageInput.pressSequentially('$');
const searchInput = page.getByPlaceholder(SKILL_PICKER_PLACEHOLDER);
await expect(searchInput).toBeVisible();
return searchInput;
}
function skillPickerOption(page: Page, name: string): Locator {
return page.locator('button[id^="skill-item-"]').filter({ hasText: name });
}
async function expectSkillInPicker(page: Page, searchInput: Locator, name: string): Promise<void> {
await searchInput.fill(name);
await expect(skillPickerOption(page, name)).toHaveCount(1);
await expect(skillPickerOption(page, name)).toBeVisible();
}
async function expectSkillAbsentFromPicker(
page: Page,
searchInput: Locator,
name: string,
): Promise<void> {
await searchInput.fill(name);
await expect(
searchInput.locator('..').getByText(NO_SKILLS_FOUND_TEXT, { exact: true }),
).toBeVisible({ timeout: 30000 });
await expect(skillPickerOption(page, name)).toHaveCount(0);
}
async function closeSkillPicker(page: Page, searchInput: Locator): Promise<void> {
await searchInput.press('Escape');
await expect(searchInput).toBeHidden();
await page.getByRole('textbox', { name: 'Message input' }).fill('');
}
test.describe('Agent Builder skills', () => {
test('creates an agent with deployment and inline skills, then removes and persists one', async ({
page,
}) => {
test.setTimeout(180000);
const agentName = uniqueAgentName('E2E Agent Skills');
const agentDescription = 'Agent Builder skill selection end-to-end coverage.';
const inlineSkillName = uniqueSkillName('e2e-agent-inline');
const unselectedSkillName = uniqueSkillName('e2e-agent-unselected');
let createdAgentId: string | undefined;
const createdSkillIds: string[] = [];
await page.goto('/c/new', { timeout: 10000 });
const token = await getAccessToken(page);
try {
const deploymentSkill = await getDeploymentSkill(page, token);
const [inlineSkill] = await createInlineSkills(
page,
token,
[inlineSkillName, unselectedSkillName],
(skill) => createdSkillIds.push(skill._id),
);
let form = await openAgentBuilder(page);
await form.getByLabel('Agent name').fill(agentName);
await form.getByLabel('Agent description').fill(agentDescription);
await selectMockModel(page, true);
form = page.getByRole('form', { name: 'Agent configuration form' });
const dialog = await openSkillsDialog(page, form);
await selectSkill(dialog, DEPLOYMENT_SKILL_NAME);
await selectSkill(dialog, inlineSkillName);
await page.keyboard.press('Escape');
await expect(dialog).toBeHidden();
const createResponsePromise = waitForAgentMutation(page, 'POST');
await form.getByRole('button', { name: 'Create', exact: true }).click();
const createResponse = await createResponsePromise;
expect(createResponse.status(), await createResponse.text()).toBe(201);
const createdAgent = (await createResponse.json()) as SkillAgentDetail;
createdAgentId = createdAgent.id;
const createPayload = createResponse.request().postDataJSON() as AgentSkillPayload;
expect(createPayload.skills_enabled).toBe(true);
expect(createPayload.skills).toHaveLength(2);
expect(createPayload.skills).toEqual(
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
);
expect(createdAgent).toMatchObject({
id: createdAgentId,
name: agentName,
skills_enabled: true,
});
expect(createdAgent.skills).toEqual(
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
);
const persistedAgent = (await waitForPersistedAgent(
page,
agentName,
agentDescription,
)) as SkillAgentDetail;
expect(persistedAgent.skills_enabled).toBe(true);
expect(persistedAgent.skills).toEqual(
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeVisible();
await expect(form.getByText(inlineSkillName, { exact: true })).toBeVisible();
const retainedDescription = `${agentDescription} Saved with both skills still selected.`;
await form.getByLabel('Agent description').fill(retainedDescription);
const retentionResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
await form.getByRole('button', { name: 'Save', exact: true }).click();
const retentionResponse = await retentionResponsePromise;
expect(retentionResponse.status(), await retentionResponse.text()).toBe(200);
const retainedAgent = (await retentionResponse.json()) as SkillAgentDetail;
const retentionPayload = retentionResponse.request().postDataJSON() as AgentSkillPayload;
expect(retentionPayload.skills_enabled).toBe(true);
expect(retentionPayload.skills).toHaveLength(2);
expect(retentionPayload.skills).toEqual(
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
);
expect(retainedAgent.skills_enabled).toBe(true);
expect(retainedAgent.skills).toHaveLength(2);
expect(retainedAgent.skills).toEqual(
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
);
const expandedRetainedAgent = await fetchJson<SkillAgentDetail>(
page,
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
token,
);
expect(expandedRetainedAgent.skills_enabled).toBe(true);
expect(expandedRetainedAgent.skills).toHaveLength(2);
expect(expandedRetainedAgent.skills).toEqual(
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeVisible();
await expect(form.getByText(inlineSkillName, { exact: true })).toBeVisible();
await form.getByRole('button', { name: 'Select Agent' }).click();
const pickerSearch = await openSkillPicker(page);
await expectSkillInPicker(page, pickerSearch, DEPLOYMENT_SKILL_NAME);
await expectSkillInPicker(page, pickerSearch, inlineSkillName);
await expectSkillAbsentFromPicker(page, pickerSearch, unselectedSkillName);
await closeSkillPicker(page, pickerSearch);
const runtimeResponse = await sendMessage(
page,
[
`${SKILL_ASSERTION_MARKER}*${DEPLOYMENT_SKILL_NAME},${inlineSkillName},!${unselectedSkillName}`,
'Verify the persisted Agent Builder skill allowlist at runtime.',
].join('\n'),
);
expect(runtimeResponse.ok()).toBeTruthy();
await expect(
page
.getByTestId('messages-view')
.getByText(`${SKILL_ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}, ${inlineSkillName}`),
).toBeVisible({ timeout: 30000 });
const inlineToolResponse = await sendMessage(
page,
`${SKILL_TOOL_INVOCATION_MARKER}${inlineSkillName}`,
);
expect(inlineToolResponse.ok()).toBeTruthy();
await expect(
page
.getByTestId('messages-view')
.getByText(`${SKILL_TOOL_ASSERTION_FINAL_TEXT}: ${inlineSkillName}`),
).toBeVisible({ timeout: 30000 });
const deploymentToolResponse = await sendMessage(
page,
`${SKILL_TOOL_INVOCATION_MARKER}${DEPLOYMENT_SKILL_NAME}`,
);
expect(deploymentToolResponse.ok()).toBeTruthy();
await expect(
page
.getByTestId('messages-view')
.getByText(`${SKILL_TOOL_ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}`),
).toBeVisible({ timeout: 30000 });
const manualPickerSearch = await openSkillPicker(page);
await expectSkillInPicker(page, manualPickerSearch, inlineSkillName);
await skillPickerOption(page, inlineSkillName).click();
await expect(manualPickerSearch).toBeHidden();
await expect(
page
.getByRole('list', { name: 'Skills queued for next submission' })
.getByText(inlineSkillName, { exact: true }),
).toBeVisible();
const manualResponse = await sendMessage(
page,
`${MANUAL_SKILL_ASSERTION_MARKER}${inlineSkillName}`,
);
expect(manualResponse.ok()).toBeTruthy();
expect((manualResponse.request().postDataJSON() as AgentSkillPayload).manualSkills).toEqual([
inlineSkillName,
]);
await expect(
page
.getByTestId('messages-view')
.getByText(`${MANUAL_SKILL_ASSERTION_FINAL_TEXT}: ${inlineSkillName}`),
).toBeVisible({ timeout: 30000 });
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
const deploymentRow = form.locator('li').filter({ hasText: DEPLOYMENT_SKILL_NAME });
await expect(deploymentRow).toHaveCount(1);
await deploymentRow.hover();
await deploymentRow.getByRole('button', { name: 'Remove from agent' }).click();
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeHidden();
const updateResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
await form.getByRole('button', { name: 'Save', exact: true }).click();
const updateResponse = await updateResponsePromise;
expect(updateResponse.status(), await updateResponse.text()).toBe(200);
const updatePayload = updateResponse.request().postDataJSON() as AgentSkillPayload;
expect(updatePayload).toMatchObject({
skills: [inlineSkill._id],
skills_enabled: true,
});
const updatedAgent = await fetchJson<SkillAgentDetail>(
page,
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
token,
);
expect(updatedAgent.skills).toEqual([inlineSkill._id]);
expect(updatedAgent.skills_enabled).toBe(true);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByText(inlineSkillName, { exact: true })).toBeVisible();
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeHidden();
const finalSkillRow = form.locator('li').filter({ hasText: inlineSkillName });
await expect(finalSkillRow).toHaveCount(1);
await finalSkillRow.hover();
await finalSkillRow.getByRole('button', { name: 'Remove from agent' }).click();
await expect(form.getByText(inlineSkillName, { exact: true })).toBeHidden();
const disableResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
await form.getByRole('button', { name: 'Save', exact: true }).click();
const disableResponse = await disableResponsePromise;
expect(disableResponse.status(), await disableResponse.text()).toBe(200);
expect(disableResponse.request().postDataJSON()).toMatchObject({
skills: [],
skills_enabled: false,
});
const disabledAgent = await fetchJson<SkillAgentDetail>(
page,
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
token,
);
expect(disabledAgent.skills).toEqual([]);
expect(disabledAgent.skills_enabled).toBe(false);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeHidden();
await expect(form.getByText(inlineSkillName, { exact: true })).toBeHidden();
await form.getByRole('button', { name: 'Select Agent' }).click();
const disabledPickerSearch = await openSkillPicker(page);
await expectSkillAbsentFromPicker(page, disabledPickerSearch, DEPLOYMENT_SKILL_NAME);
await expectSkillAbsentFromPicker(page, disabledPickerSearch, inlineSkillName);
await closeSkillPicker(page, disabledPickerSearch);
const disabledRuntimeResponse = await sendMessage(
page,
`${SKILL_ASSERTION_MARKER}!${DEPLOYMENT_SKILL_NAME},!${inlineSkillName}`,
);
expect(disabledRuntimeResponse.ok()).toBeTruthy();
await expect(
page.getByTestId('messages-view').getByText(`${SKILL_ASSERTION_FINAL_TEXT}: none`),
).toBeVisible({ timeout: 30000 });
} finally {
await settleCleanup([
cleanupAgent(page, createdAgentId),
deleteSkills(page, token, createdSkillIds),
]);
}
});
test('persists explicit use-all semantics and includes skills created later', async ({
page,
}) => {
test.setTimeout(180000);
const agentName = uniqueAgentName('E2E Agent All Skills');
const agentDescription = 'Agent Builder explicit use-all-skills end-to-end coverage.';
const futureSkillName = uniqueSkillName('e2e-agent-future');
let createdAgentId: string | undefined;
let futureSkill: SkillSummary | undefined;
await page.goto('/c/new', { timeout: 10000 });
const token = await getAccessToken(page);
try {
let form = await openAgentBuilder(page);
await form.getByLabel('Agent name').fill(agentName);
await form.getByLabel('Agent description').fill(agentDescription);
await selectMockModel(page, true);
form = page.getByRole('form', { name: 'Agent configuration form' });
const useAllSwitch = form.getByRole('switch', { name: 'Use all skills' });
await expect(useAllSwitch).toHaveAttribute('aria-checked', 'false');
await useAllSwitch.click();
await expect(useAllSwitch).toHaveAttribute('aria-checked', 'true');
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toHaveCount(0);
const createResponsePromise = waitForAgentMutation(page, 'POST');
await form.getByRole('button', { name: 'Create', exact: true }).click();
const createResponse = await createResponsePromise;
expect(createResponse.status(), await createResponse.text()).toBe(201);
const createdAgent = (await createResponse.json()) as SkillAgentDetail;
createdAgentId = createdAgent.id;
const createPayload = createResponse.request().postDataJSON() as AgentSkillPayload;
expect(createPayload).toMatchObject({
skills: [],
skills_enabled: true,
});
expect(createdAgent).toMatchObject({
id: createdAgentId,
skills: [],
skills_enabled: true,
});
const persistedAgent = (await waitForPersistedAgent(
page,
agentName,
agentDescription,
)) as SkillAgentDetail;
expect(persistedAgent.skills).toEqual([]);
expect(persistedAgent.skills_enabled).toBe(true);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByRole('switch', { name: 'Use all skills' })).toHaveAttribute(
'aria-checked',
'true',
);
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toHaveCount(0);
futureSkill = await createInlineSkill(page, token, futureSkillName);
await form.getByRole('button', { name: 'Select Agent' }).click();
const useAllPickerSearch = await openSkillPicker(page);
await expectSkillInPicker(page, useAllPickerSearch, futureSkillName);
await closeSkillPicker(page, useAllPickerSearch);
const runtimeResponse = await sendMessage(
page,
[
`${SKILL_ASSERTION_MARKER}${DEPLOYMENT_SKILL_NAME},${futureSkillName}`,
'Verify use-all includes a skill created after the agent was saved.',
].join('\n'),
);
expect(runtimeResponse.ok()).toBeTruthy();
await expect(
page
.getByTestId('messages-view')
.getByText(`${SKILL_ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}, ${futureSkillName}`),
).toBeVisible({ timeout: 30000 });
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
const persistedUseAllSwitch = form.getByRole('switch', { name: 'Use all skills' });
await persistedUseAllSwitch.click();
await expect(persistedUseAllSwitch).toHaveAttribute('aria-checked', 'false');
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toBeVisible();
const updateResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
await form.getByRole('button', { name: 'Save', exact: true }).click();
const updateResponse = await updateResponsePromise;
expect(updateResponse.status(), await updateResponse.text()).toBe(200);
expect(updateResponse.request().postDataJSON()).toMatchObject({
skills: [],
skills_enabled: false,
});
const disabledAgent = await fetchJson<SkillAgentDetail>(
page,
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
token,
);
expect(disabledAgent.skills).toEqual([]);
expect(disabledAgent.skills_enabled).toBe(false);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByRole('switch', { name: 'Use all skills' })).toHaveAttribute(
'aria-checked',
'false',
);
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toBeVisible();
await form.getByRole('button', { name: 'Select Agent' }).click();
const disabledPickerSearch = await openSkillPicker(page);
await expectSkillAbsentFromPicker(page, disabledPickerSearch, futureSkillName);
await closeSkillPicker(page, disabledPickerSearch);
const disabledRuntimeResponse = await sendMessage(
page,
[
`${SKILL_ASSERTION_MARKER}!${DEPLOYMENT_SKILL_NAME},!${futureSkillName}`,
'Verify the master skill toggle hides every skill at runtime.',
].join('\n'),
);
expect(disabledRuntimeResponse.ok()).toBeTruthy();
await expect(
page.getByTestId('messages-view').getByText(`${SKILL_ASSERTION_FINAL_TEXT}: none`),
).toBeVisible({ timeout: 30000 });
} finally {
await settleCleanup([
cleanupAgent(page, createdAgentId),
...(futureSkill ? [deleteSkill(page, token, futureSkill._id)] : []),
]);
}
});
test('creates a skill inside the builder and auto-selects it for the new agent', async ({
page,
}) => {
test.setTimeout(180000);
const agentName = uniqueAgentName('E2E Agent Created Skill');
const agentDescription = 'Agent Builder inline skill creation end-to-end coverage.';
const skillName = uniqueSkillName('e2e-builder-created');
let createdAgentId: string | undefined;
let createdSkill: SkillSummary | undefined;
let agentPostCount = 0;
const countAgentPosts = (request: Request) => {
if (request.method() === 'POST' && new URL(request.url()).pathname === '/api/agents') {
agentPostCount += 1;
}
};
page.on('request', countAgentPosts);
await page.goto('/c/new', { timeout: 10000 });
const token = await getAccessToken(page);
try {
let form = await openAgentBuilder(page);
await form.getByLabel('Agent name').fill(agentName);
await form.getByLabel('Agent description').fill(agentDescription);
await selectMockModel(page, true);
form = page.getByRole('form', { name: 'Agent configuration form' });
const skillsDialog = await openSkillsDialog(page, form);
await skillsDialog.getByRole('button', { name: 'Create Skill', exact: true }).click();
const createSkillDialog = page
.getByRole('dialog')
.filter({ hasText: 'Write skill instructions' });
await expect(createSkillDialog).toBeVisible();
await createSkillDialog.getByLabel('Name').fill(skillName);
await createSkillDialog.getByLabel('Description').fill(INLINE_SKILL_DESCRIPTION);
await createSkillDialog
.getByLabel('Instructions')
.fill(`# ${skillName}\n\nCreated inline from Agent Builder.`);
const skillResponsePromise = page.waitForResponse(
(response) =>
response.request().method() === 'POST' &&
new URL(response.url()).pathname === '/api/skills',
{ timeout: 30000 },
);
await createSkillDialog.getByRole('button', { name: 'Create', exact: true }).click();
const skillResponse = await skillResponsePromise;
expect(skillResponse.status(), await skillResponse.text()).toBe(201);
createdSkill = (await skillResponse.json()) as SkillSummary;
expect(skillResponse.request().postDataJSON()).toMatchObject({
name: skillName,
description: INLINE_SKILL_DESCRIPTION,
});
expect(agentPostCount).toBe(0);
await expect(createSkillDialog).toBeHidden();
const createdCard = await skillCard(skillsDialog, skillName);
await expect(createdCard).toHaveAttribute('aria-pressed', 'true');
await page.keyboard.press('Escape');
await expect(skillsDialog).toBeHidden();
await expect(form.getByText(skillName, { exact: true })).toBeVisible();
const createAgentResponsePromise = waitForAgentMutation(page, 'POST');
await form.getByRole('button', { name: 'Create', exact: true }).click();
const createAgentResponse = await createAgentResponsePromise;
expect(createAgentResponse.status(), await createAgentResponse.text()).toBe(201);
expect(agentPostCount).toBe(1);
const createdAgent = (await createAgentResponse.json()) as SkillAgentDetail;
createdAgentId = createdAgent.id;
expect(createAgentResponse.request().postDataJSON()).toMatchObject({
skills: [createdSkill._id],
skills_enabled: true,
});
expect(createdAgent).toMatchObject({
id: createdAgentId,
skills: [createdSkill._id],
skills_enabled: true,
});
const persistedAgent = (await waitForPersistedAgent(
page,
agentName,
agentDescription,
)) as SkillAgentDetail;
expect(persistedAgent.skills).toEqual([createdSkill._id]);
expect(persistedAgent.skills_enabled).toBe(true);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByText(skillName, { exact: true })).toBeVisible();
} finally {
page.off('request', countAgentPosts);
await settleCleanup([
cleanupAgent(page, createdAgentId),
...(createdSkill ? [deleteSkill(page, token, createdSkill._id)] : []),
]);
}
});
test('discovers and selects a skill beyond the first 100 accessible skills', async ({ page }) => {
test.setTimeout(240000);
const agentName = uniqueAgentName('E2E Agent Paginated Skill');
const agentDescription = 'Agent Builder paginated skill discovery end-to-end coverage.';
const targetSkillName = uniqueSkillName('e2e-agent-off-page');
const fillerPrefix = uniqueSkillName('e2e-agent-page');
let createdAgentId: string | undefined;
const createdSkillIds: string[] = [];
await page.goto('/c/new', { timeout: 10000 });
const token = await getAccessToken(page);
try {
const targetSkill = await createInlineSkill(page, token, targetSkillName);
createdSkillIds.push(targetSkill._id);
const fillerNames = Array.from(
{ length: 100 },
(_, index) => `${fillerPrefix}-${String(index).padStart(3, '0')}`,
);
await createInlineSkills(page, token, fillerNames, (skill) =>
createdSkillIds.push(skill._id),
);
const firstPage = await fetchJson<{
skills: SkillSummary[];
has_more: boolean;
after: string | null;
}>(page, '/api/skills?limit=100', token);
expect(firstPage.has_more).toBe(true);
expect(firstPage.after).toBeTruthy();
expect(firstPage.skills.some((skill) => skill._id === targetSkill._id)).toBe(false);
let form = await openAgentBuilder(page);
await form.getByLabel('Agent name').fill(agentName);
await form.getByLabel('Agent description').fill(agentDescription);
await selectMockModel(page, true);
form = page.getByRole('form', { name: 'Agent configuration form' });
const nextPageResponsePromise = page.waitForResponse(
(response) => {
const url = new URL(response.url());
return url.pathname === '/api/skills' && url.searchParams.has('cursor');
},
{ timeout: 30000 },
);
const dialog = await openSkillsDialog(page, form);
const nextPageResponse = await nextPageResponsePromise;
expect(nextPageResponse.ok(), await nextPageResponse.text()).toBe(true);
await dialog.getByRole('textbox', { name: 'Search skills...' }).fill(targetSkillName);
await selectSkill(dialog, targetSkillName);
await page.keyboard.press('Escape');
await expect(dialog).toBeHidden();
await expect(form.getByText(targetSkillName, { exact: true })).toBeVisible();
const createResponsePromise = waitForAgentMutation(page, 'POST');
await form.getByRole('button', { name: 'Create', exact: true }).click();
const createResponse = await createResponsePromise;
expect(createResponse.status(), await createResponse.text()).toBe(201);
const createdAgent = (await createResponse.json()) as SkillAgentDetail;
createdAgentId = createdAgent.id;
expect(createResponse.request().postDataJSON()).toMatchObject({
skills: [targetSkill._id],
skills_enabled: true,
});
const persistedAgent = (await waitForPersistedAgent(
page,
agentName,
agentDescription,
)) as SkillAgentDetail;
expect(persistedAgent.skills).toEqual([targetSkill._id]);
expect(persistedAgent.skills_enabled).toBe(true);
form = await openAgentBuilder(page);
await selectAgent(page, form, agentName);
await expect(form.getByText(targetSkillName, { exact: true })).toBeVisible();
} finally {
await settleCleanup([
cleanupAgent(page, createdAgentId),
deleteSkills(page, token, createdSkillIds),
]);
}
});
});

View file

@ -12,9 +12,11 @@ import {
} from './helpers';
const MODEL_SPEC_LABEL = 'E2E Skill Scope';
const ASSERTION_MARKER = 'E2E_ASSERT_MODEL_SPEC_SKILLS';
const ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed';
const ASSERTION_MARKER = 'E2E_ASSERT_SKILLS:';
const ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
const ACCESSIBLE_SKILL_NAME = 'e2e-model-spec-allowed';
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
const MISSING_SKILL_NAME = 'e2e-model-spec-missing';
const INACCESSIBLE_SKILL_NAME = 'e2e-model-spec-inaccessible';
const ALWAYS_APPLY_BODY_MARKER = 'E2E_ALWAYS_APPLY_BODY_MARKER';
const INACCESSIBLE_AUTHOR_ID = new ObjectId('64f000000000000000000001');
@ -93,7 +95,15 @@ async function seedAccessibleSkill(page: Page, token: string): Promise<SkillDeta
});
}
async function seedInaccessibleSkill() {
async function deleteAccessibleSkill(page: Page, token: string, skillId: string) {
await requestJson<{ deleted: boolean }>(page, {
path: `/api/skills/${encodeURIComponent(skillId)}`,
token,
method: 'DELETE',
});
}
async function seedInaccessibleSkill(): Promise<string> {
applyRuntimeEnv();
if (!process.env.MONGO_URI) {
throw new Error('MONGO_URI must be available for model-spec skill mock e2e tests');
@ -141,7 +151,30 @@ async function seedInaccessibleSkill() {
});
if (skill?._id) {
await aclEntries.deleteMany({ resourceType: 'skill', resourceId: skill._id });
return skill._id.toString();
}
throw new Error(`Failed to seed inaccessible skill "${INACCESSIBLE_SKILL_NAME}"`);
} finally {
await client.close();
}
}
async function deleteInaccessibleSkill(skillId: string) {
applyRuntimeEnv();
if (!process.env.MONGO_URI) {
throw new Error('MONGO_URI must be available for model-spec skill mock e2e tests');
}
const client = new MongoClient(process.env.MONGO_URI);
await client.connect();
try {
const db = client.db();
const objectId = new ObjectId(skillId);
await db.collection('aclentries').deleteMany({
resourceType: 'skill',
resourceId: objectId,
});
await db.collection('skills').deleteOne({ _id: objectId });
} finally {
await client.close();
}
@ -155,21 +188,60 @@ test.describe('model spec skills', () => {
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
const token = await getAccessToken(page);
const skill = await seedAccessibleSkill(page, token);
expect(skill.alwaysApply).toBe(true);
await seedInaccessibleSkill();
let accessibleSkillId: string | undefined;
let inaccessibleSkillId: string | undefined;
let testFailure: unknown;
await selectModelSpec(page, MODEL_SPEC_LABEL);
const response = await sendMessage(
page,
`${ASSERTION_MARKER}\nVerify model-spec skill scope and always-apply frontmatter.`,
);
expect(response.ok()).toBeTruthy();
try {
const skill = await seedAccessibleSkill(page, token);
accessibleSkillId = skill._id;
expect(skill.alwaysApply).toBe(true);
inaccessibleSkillId = await seedInaccessibleSkill();
await expect(
page
.getByTestId('messages-view')
.getByText(`${ASSERTION_FINAL_TEXT}: ${ACCESSIBLE_SKILL_NAME}`),
).toBeVisible({ timeout: 30000 });
await selectModelSpec(page, MODEL_SPEC_LABEL);
const response = await sendMessage(
page,
[
`${ASSERTION_MARKER}*${ACCESSIBLE_SKILL_NAME},*${DEPLOYMENT_SKILL_NAME},!${MISSING_SKILL_NAME},!${INACCESSIBLE_SKILL_NAME}`,
'Verify model-spec skill scope and always-apply frontmatter.',
].join('\n'),
);
expect(response.ok()).toBeTruthy();
await expect(
page
.getByTestId('messages-view')
.getByText(`${ASSERTION_FINAL_TEXT}: ${ACCESSIBLE_SKILL_NAME}, ${DEPLOYMENT_SKILL_NAME}`),
).toBeVisible({ timeout: 30000 });
} catch (error) {
testFailure = error;
} finally {
const cleanupTasks: Promise<unknown>[] = [];
if (accessibleSkillId) {
cleanupTasks.push(deleteAccessibleSkill(page, token, accessibleSkillId));
}
if (inaccessibleSkillId) {
cleanupTasks.push(deleteInaccessibleSkill(inaccessibleSkillId));
}
const cleanupResults = await Promise.allSettled(cleanupTasks);
const cleanupFailures = cleanupResults
.filter((result): result is PromiseRejectedResult => result.status === 'rejected')
.map((result) => result.reason);
if (testFailure && cleanupFailures.length > 0) {
testFailure = new AggregateError(
[testFailure, ...cleanupFailures],
'The model-spec skill assertion and fixture cleanup both failed',
);
} else if (cleanupFailures.length > 0) {
testFailure = new AggregateError(
cleanupFailures,
'Model-spec skill fixture cleanup failed',
);
}
}
if (testFailure) {
throw testFailure;
}
});
});