mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-10-21 01:22:54 +00:00
🧩 fix: Harden Agent Skill Lifecycles End to End (#14429)
Some checks failed
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Has been cancelled
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Has been cancelled
GitNexus Index / index (push) Has been cancelled
GitNexus Index / post-index (push) Has been cancelled
Some checks failed
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Has been cancelled
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Has been cancelled
GitNexus Index / index (push) Has been cancelled
GitNexus Index / post-index (push) Has been cancelled
* test: cover agent skill lifecycles end to end * style: sort agent skill imports
This commit is contained in:
parent
73699b5c25
commit
f3159f9891
19 changed files with 2160 additions and 116 deletions
|
|
@ -10,7 +10,7 @@ const {
|
|||
const { isEphemeralAgentId } = require('librechat-data-provider');
|
||||
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
|
||||
const { getMCPServerTools } = require('~/server/services/Config');
|
||||
const { canAuthorSkillFiles } = require('./skillDeps');
|
||||
const { getSkillDbMethods, canAuthorSkillFiles } = require('./skillDeps');
|
||||
const db = require('~/models');
|
||||
|
||||
const loadAddedAgent = (params) =>
|
||||
|
|
@ -97,6 +97,7 @@ const processAddedConvo = async ({
|
|||
});
|
||||
|
||||
try {
|
||||
const skillDbMethods = getSkillDbMethods();
|
||||
const addedAgent = await loadAddedAgent({ req, conversation: addedConvo, primaryAgent });
|
||||
if (!addedAgent) {
|
||||
return { userMCPAuthMap };
|
||||
|
|
@ -138,7 +139,7 @@ const processAddedConvo = async ({
|
|||
const resolvedSkillIds = await resolveModelSpecSkillIds({
|
||||
names: selectedModelSpec.skills,
|
||||
accessibleSkillIds,
|
||||
getSkillByName: db.getSkillByName,
|
||||
getSkillByName: skillDbMethods.getSkillByName,
|
||||
});
|
||||
addedAgent.skills_enabled = true;
|
||||
addedAgent.skills = resolvedSkillIds.map((id) => id.toString());
|
||||
|
|
@ -195,9 +196,9 @@ const processAddedConvo = async ({
|
|||
getToolFilesByIds: db.getToolFilesByIds,
|
||||
getCodeGeneratedFiles: db.getCodeGeneratedFiles,
|
||||
filterFilesByAgentAccess,
|
||||
listSkillsByAccess: db.listSkillsByAccess,
|
||||
listAlwaysApplySkills: db.listAlwaysApplySkills,
|
||||
getSkillByName: db.getSkillByName,
|
||||
listSkillsByAccess: skillDbMethods.listSkillsByAccess,
|
||||
listAlwaysApplySkills: skillDbMethods.listAlwaysApplySkills,
|
||||
getSkillByName: skillDbMethods.getSkillByName,
|
||||
},
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,12 @@ const mockLoadAddedAgent = jest.fn();
|
|||
const mockResolveAgentScopedSkillIds = jest.fn();
|
||||
const mockResolveModelSpecSkillIds = jest.fn();
|
||||
const mockCanAuthorSkillFiles = jest.fn();
|
||||
const mockGetSkillDbMethods = jest.fn();
|
||||
const mockGetAgent = jest.fn();
|
||||
const mockGetMCPServerTools = jest.fn();
|
||||
const mockRegistryGetSkillByName = jest.fn();
|
||||
const mockRegistryListSkillsByAccess = jest.fn();
|
||||
const mockRegistryListAlwaysApplySkills = jest.fn();
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: {
|
||||
|
|
@ -35,6 +39,7 @@ jest.mock('~/server/services/Config', () => ({
|
|||
|
||||
jest.mock('./skillDeps', () => ({
|
||||
canAuthorSkillFiles: (...args) => mockCanAuthorSkillFiles(...args),
|
||||
getSkillDbMethods: () => mockGetSkillDbMethods(),
|
||||
}));
|
||||
|
||||
jest.mock('~/models', () => ({
|
||||
|
|
@ -45,7 +50,6 @@ jest.mock('~/models', () => ({
|
|||
}));
|
||||
|
||||
const { processAddedConvo } = require('./addedConvo');
|
||||
const db = require('~/models');
|
||||
const { Constants } = require('librechat-data-provider');
|
||||
|
||||
const makeReq = () => ({ user: { id: 'u1', role: 'USER' } });
|
||||
|
|
@ -72,6 +76,11 @@ describe('processAddedConvo', () => {
|
|||
);
|
||||
mockResolveModelSpecSkillIds.mockResolvedValue([]);
|
||||
mockCanAuthorSkillFiles.mockReturnValue(false);
|
||||
mockGetSkillDbMethods.mockReturnValue({
|
||||
getSkillByName: mockRegistryGetSkillByName,
|
||||
listSkillsByAccess: mockRegistryListSkillsByAccess,
|
||||
listAlwaysApplySkills: mockRegistryListAlwaysApplySkills,
|
||||
});
|
||||
});
|
||||
|
||||
const baseParams = (overrides = {}) => ({
|
||||
|
|
@ -125,6 +134,75 @@ describe('processAddedConvo', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('keeps deployment-aware skill metadata on a persisted added-agent config', async () => {
|
||||
const deploymentSkillId = { toString: () => 'deployment-skill' };
|
||||
const agentConfigs = new Map();
|
||||
const initializedConfig = {
|
||||
id: 'persisted-added-agent',
|
||||
additional_instructions: '<skill_catalog>deployment-skill</skill_catalog>',
|
||||
manualSkillPrimes: [],
|
||||
alwaysApplySkillPrimes: [
|
||||
{
|
||||
_id: 'deployment-skill',
|
||||
name: 'deployment-skill',
|
||||
body: 'deployment skill body',
|
||||
},
|
||||
],
|
||||
toolDefinitions: [{ name: 'skill' }],
|
||||
userMCPAuthMap: undefined,
|
||||
};
|
||||
|
||||
mockLoadAddedAgent.mockResolvedValue({
|
||||
id: 'persisted-added-agent',
|
||||
provider: 'openai',
|
||||
skills_enabled: true,
|
||||
skills: ['deployment-skill'],
|
||||
});
|
||||
mockResolveAgentScopedSkillIds.mockReturnValue([deploymentSkillId]);
|
||||
mockInitializeAgent.mockResolvedValue(initializedConfig);
|
||||
|
||||
await processAddedConvo(
|
||||
baseParams({
|
||||
accessibleSkillIds: [deploymentSkillId],
|
||||
editableSkillIds: [deploymentSkillId],
|
||||
skillsCapabilityEnabled: true,
|
||||
agentConfigs,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockResolveModelSpecSkillIds).not.toHaveBeenCalled();
|
||||
expect(mockInitializeAgent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
agent: expect.objectContaining({
|
||||
id: 'persisted-added-agent',
|
||||
skills_enabled: true,
|
||||
skills: ['deployment-skill'],
|
||||
}),
|
||||
accessibleSkillIds: [deploymentSkillId],
|
||||
}),
|
||||
expect.objectContaining({
|
||||
listSkillsByAccess: mockRegistryListSkillsByAccess,
|
||||
listAlwaysApplySkills: mockRegistryListAlwaysApplySkills,
|
||||
getSkillByName: mockRegistryGetSkillByName,
|
||||
}),
|
||||
);
|
||||
expect(agentConfigs.get('persisted-added-agent')).toBe(initializedConfig);
|
||||
expect(agentConfigs.get('persisted-added-agent')).toEqual(
|
||||
expect.objectContaining({
|
||||
additional_instructions: '<skill_catalog>deployment-skill</skill_catalog>',
|
||||
manualSkillPrimes: [],
|
||||
alwaysApplySkillPrimes: [
|
||||
expect.objectContaining({
|
||||
name: 'deployment-skill',
|
||||
body: 'deployment skill body',
|
||||
}),
|
||||
],
|
||||
toolDefinitions: [expect.objectContaining({ name: 'skill' })],
|
||||
}),
|
||||
);
|
||||
expect(mockGetSkillDbMethods).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('resolves and forwards model-spec skill scope for added ephemeral agents', async () => {
|
||||
const accessibleSkillId = { toString: () => 'accessible-skill' };
|
||||
const editableSkillId = { toString: () => 'editable-skill' };
|
||||
|
|
@ -181,7 +259,7 @@ describe('processAddedConvo', () => {
|
|||
expect(mockResolveModelSpecSkillIds).toHaveBeenCalledWith({
|
||||
names: ['finance-analyst'],
|
||||
accessibleSkillIds: [accessibleSkillId],
|
||||
getSkillByName: db.getSkillByName,
|
||||
getSkillByName: mockRegistryGetSkillByName,
|
||||
});
|
||||
expect(mockResolveAgentScopedSkillIds).toHaveBeenNthCalledWith(1, {
|
||||
agent: expect.objectContaining({
|
||||
|
|
@ -222,10 +300,11 @@ describe('processAddedConvo', () => {
|
|||
defaultActiveOnShare: true,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
listSkillsByAccess: db.listSkillsByAccess,
|
||||
listAlwaysApplySkills: db.listAlwaysApplySkills,
|
||||
getSkillByName: db.getSkillByName,
|
||||
listSkillsByAccess: mockRegistryListSkillsByAccess,
|
||||
listAlwaysApplySkills: mockRegistryListAlwaysApplySkills,
|
||||
getSkillByName: mockRegistryGetSkillByName,
|
||||
}),
|
||||
);
|
||||
expect(mockGetSkillDbMethods).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -402,7 +402,7 @@ const initializeClient = async ({ req, res, signal, endpointOption, jobCreatedAt
|
|||
const resolvedSkillIds = await resolveModelSpecSkillIds({
|
||||
names: selectedModelSpec.skills,
|
||||
accessibleSkillIds,
|
||||
getSkillByName: db.getSkillByName,
|
||||
getSkillByName: skillDbMethods.getSkillByName,
|
||||
});
|
||||
primaryAgent.skills_enabled = true;
|
||||
primaryAgent.skills = resolvedSkillIds.map((id) => id.toString());
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ jest.mock('~/cache', () => ({
|
|||
}));
|
||||
|
||||
const { initializeClient } = require('./initialize');
|
||||
const { getSkillToolDeps } = require('./skillDeps');
|
||||
const { getSkillDbMethods, getSkillToolDeps } = require('./skillDeps');
|
||||
const { getModelsConfig } = require('~/server/controllers/ModelController');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { User, AclEntry } = require('~/db/models');
|
||||
|
|
@ -360,6 +360,63 @@ describe('initializeClient — processAgent ACL gate', () => {
|
|||
canCreateSkillSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('resolves model-spec skill names through deployment-aware skill methods', async () => {
|
||||
const deploymentSkillId = new mongoose.Types.ObjectId();
|
||||
await AclEntry.create({
|
||||
principalType: PrincipalType.USER,
|
||||
principalId: testUser._id,
|
||||
principalModel: PrincipalModel.USER,
|
||||
resourceType: ResourceType.SKILL,
|
||||
resourceId: new mongoose.Types.ObjectId(),
|
||||
permBits: PermissionBits.VIEW,
|
||||
grantedBy: testUser._id,
|
||||
});
|
||||
const endpointOption = makeEndpointOption();
|
||||
endpointOption.spec = 'spec-deployment-skill';
|
||||
endpointOption.agent = Promise.resolve({
|
||||
id: Constants.EPHEMERAL_AGENT_ID,
|
||||
name: 'Ephemeral Primary',
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
tools: [],
|
||||
});
|
||||
mockInitializeAgent.mockResolvedValue(makePrimaryConfig([]));
|
||||
const req = makeReq();
|
||||
req.config.endpoints.agents = { capabilities: ['skills'] };
|
||||
req.config.modelSpecs = {
|
||||
list: [{ name: 'spec-deployment-skill', skills: ['deployment-skill'] }],
|
||||
};
|
||||
const getSkillByNameSpy = jest.spyOn(getSkillDbMethods(), 'getSkillByName').mockResolvedValue({
|
||||
_id: deploymentSkillId,
|
||||
name: 'deployment-skill',
|
||||
source: 'deployment',
|
||||
});
|
||||
const canCreateSkillSpy = jest
|
||||
.spyOn(getSkillToolDeps(), 'canCreateSkill')
|
||||
.mockResolvedValue(false);
|
||||
|
||||
try {
|
||||
await initializeClient({
|
||||
req,
|
||||
res: {},
|
||||
signal: new AbortController().signal,
|
||||
endpointOption,
|
||||
});
|
||||
|
||||
expect(getSkillByNameSpy).toHaveBeenCalledWith(
|
||||
'deployment-skill',
|
||||
expect.any(Array),
|
||||
expect.any(Object),
|
||||
);
|
||||
const initializeParams = mockInitializeAgent.mock.calls[0][0];
|
||||
expect(initializeParams.agent.skills_enabled).toBe(true);
|
||||
expect(initializeParams.agent.skills).toEqual([deploymentSkillId.toString()]);
|
||||
} finally {
|
||||
getSkillByNameSpy.mockRestore();
|
||||
canCreateSkillSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('initializeClient — subagent loading', () => {
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ import type { TSkillSummary } from 'librechat-data-provider';
|
|||
import type { MentionOption } from '~/common';
|
||||
import useInitPopoverInput from '~/hooks/Input/useInitPopoverInput';
|
||||
import { useLocalize, useSkillActiveState } from '~/hooks';
|
||||
import { useAgentsMapContext } from '~/Providers';
|
||||
import { useSkillsInfiniteQuery } from '~/data-provider';
|
||||
import { isEphemeralAgent } from '~/common';
|
||||
import { useAgentsMapContext } from '~/Providers';
|
||||
import { ephemeralAgentByConvoId } from '~/store';
|
||||
import { isEphemeralAgent } from '~/common';
|
||||
import { removeCharIfLast } from '~/utils';
|
||||
import MentionItem from './MentionItem';
|
||||
import store from '~/store';
|
||||
|
|
@ -50,7 +50,7 @@ export function filterSkillsForPopover(
|
|||
skills: TSkillSummary[],
|
||||
ctx: {
|
||||
agentSkillIds: string[] | null | undefined;
|
||||
isActive: (skill: Pick<TSkillSummary, '_id' | 'author'>) => boolean;
|
||||
isActive: (skill: Pick<TSkillSummary, '_id' | 'author' | 'source'>) => boolean;
|
||||
},
|
||||
): TSkillSummary[] {
|
||||
const { agentSkillIds, isActive } = ctx;
|
||||
|
|
@ -128,21 +128,10 @@ function SkillsCommandContent({
|
|||
const { data, isLoading, isError, fetchNextPage, hasNextPage, isFetchingNextPage } =
|
||||
useSkillsInfiniteQuery({ limit: 50 });
|
||||
|
||||
/* Sticky circuit breaker: once any page request fails, stop auto-fetching
|
||||
for the lifetime of the popover so a transient API error does not turn
|
||||
into an unbounded retry loop (isError can flip back to false on the
|
||||
next attempt, which would otherwise re-arm the auto-fetch effect). */
|
||||
const paginationBlockedRef = useRef(false);
|
||||
useEffect(() => {
|
||||
if (isError) {
|
||||
paginationBlockedRef.current = true;
|
||||
}
|
||||
}, [isError]);
|
||||
|
||||
/* Auto-fetch all pages so client-side search covers the full catalog,
|
||||
not just the first page. The skills API is server-side capped. */
|
||||
useEffect(() => {
|
||||
if (paginationBlockedRef.current || isError) {
|
||||
if (isError) {
|
||||
return;
|
||||
}
|
||||
if (hasNextPage && !isFetchingNextPage) {
|
||||
|
|
|
|||
|
|
@ -435,6 +435,36 @@ describe('SkillsCommand', () => {
|
|||
expect(screen.queryByRole('button', { name: /Brand Guidelines/i })).toBeNull();
|
||||
expect(screen.getByRole('button', { name: /Style Guide/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('resumes catalog pagination after an external refetch clears an error', () => {
|
||||
const fetchNextPage = jest.fn();
|
||||
mockUseSkillsInfiniteQuery.mockReturnValue({
|
||||
data: skillsResponse,
|
||||
isLoading: false,
|
||||
isError: true,
|
||||
fetchNextPage,
|
||||
hasNextPage: true,
|
||||
isFetchingNextPage: false,
|
||||
});
|
||||
|
||||
const textAreaRef = makeTextarea('$');
|
||||
const { rerender } = render(
|
||||
<SkillsCommand index={0} textAreaRef={textAreaRef} conversationId={CONVO_ID} />,
|
||||
);
|
||||
expect(fetchNextPage).not.toHaveBeenCalled();
|
||||
|
||||
mockUseSkillsInfiniteQuery.mockReturnValue({
|
||||
data: skillsResponse,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
fetchNextPage,
|
||||
hasNextPage: true,
|
||||
isFetchingNextPage: false,
|
||||
});
|
||||
rerender(<SkillsCommand index={1} textAreaRef={textAreaRef} conversationId={CONVO_ID} />);
|
||||
|
||||
expect(fetchNextPage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filterSkillsForPopover', () => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { useMemo, useState, useCallback } from 'react';
|
||||
import { useMemo, useState, useCallback, useEffect } from 'react';
|
||||
import { Plus, Search } from 'lucide-react';
|
||||
import { useFormContext, useWatch } from 'react-hook-form';
|
||||
import { PermissionTypes, Permissions } from 'librechat-data-provider';
|
||||
|
|
@ -11,7 +11,7 @@ import {
|
|||
OGDialogContent,
|
||||
OGDialogDescription,
|
||||
} from '@librechat/client';
|
||||
import type { TSkill } from 'librechat-data-provider';
|
||||
import type { TSkill, TSkillSummary } from 'librechat-data-provider';
|
||||
import type { TranslationKeys } from '~/hooks/useLocalize';
|
||||
import type { CategoryOption } from './CategoryFilter';
|
||||
import type { AgentItem } from './items/types';
|
||||
|
|
@ -19,8 +19,8 @@ import type { AgentForm } from '~/common';
|
|||
import { useLocalize, useHasAccess, useAuthContext, useToolFavorites } from '~/hooks';
|
||||
import { CreateSkillDialog } from '~/components/Skills/dialogs';
|
||||
import { skillsEnabledTransition } from './items/mutations';
|
||||
import { useSkillsInfiniteQuery } from '~/data-provider';
|
||||
import MarketplaceCatalog from './MarketplaceCatalog';
|
||||
import { useListSkillsQuery } from '~/data-provider';
|
||||
import { CategoryIcon } from '~/components/Prompts';
|
||||
import { buildSkillItems } from './items/catalog';
|
||||
import ItemDialog from './ItemDialog/ItemDialog';
|
||||
|
|
@ -55,12 +55,30 @@ export default function SkillsDialog({ open, onOpenChange, agentId }: SkillsDial
|
|||
permissionType: PermissionTypes.SKILLS,
|
||||
permission: Permissions.CREATE,
|
||||
});
|
||||
const { data: skillsData, isLoading: isLoadingSkills } = useListSkillsQuery(
|
||||
{ limit: 100 },
|
||||
{ enabled: hasSkillsAccess },
|
||||
);
|
||||
const {
|
||||
data: skillsData,
|
||||
isLoading: isLoadingSkills,
|
||||
isError: isSkillsError,
|
||||
fetchNextPage,
|
||||
refetch: refetchSkills,
|
||||
hasNextPage,
|
||||
isFetchingNextPage,
|
||||
} = useSkillsInfiniteQuery({ limit: 100 }, { enabled: hasSkillsAccess });
|
||||
const { favoriteKeys, toggle: toggleFavorite } = useToolFavorites();
|
||||
|
||||
useEffect(() => {
|
||||
if (isSkillsError) {
|
||||
return;
|
||||
}
|
||||
if (hasNextPage && !isFetchingNextPage) {
|
||||
fetchNextPage();
|
||||
}
|
||||
}, [hasNextPage, isFetchingNextPage, isSkillsError, fetchNextPage]);
|
||||
|
||||
const handleRetrySkills = useCallback(() => {
|
||||
void refetchSkills();
|
||||
}, [refetchSkills]);
|
||||
|
||||
const skillsField = useWatch({ control, name: 'skills' });
|
||||
const selectedIds = useMemo(
|
||||
() => new Set(((skillsField ?? []) as string[]).map((id) => itemKey({ kind: 'skill', id }))),
|
||||
|
|
@ -73,10 +91,22 @@ export default function SkillsDialog({ open, onOpenChange, agentId }: SkillsDial
|
|||
const [createOpen, setCreateOpen] = useState(false);
|
||||
const [detailItem, setDetailItem] = useState<AgentItem | null>(null);
|
||||
|
||||
const catalog = useMemo(
|
||||
() => buildSkillItems(skillsData?.skills ?? [], user?.id),
|
||||
[skillsData, user?.id],
|
||||
);
|
||||
const skills = useMemo(() => {
|
||||
const allSkills: TSkillSummary[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const page of skillsData?.pages ?? []) {
|
||||
for (const skill of page.skills) {
|
||||
if (seen.has(skill._id)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(skill._id);
|
||||
allSkills.push(skill);
|
||||
}
|
||||
}
|
||||
return allSkills;
|
||||
}, [skillsData?.pages]);
|
||||
|
||||
const catalog = useMemo(() => buildSkillItems(skills, user?.id), [skills, user?.id]);
|
||||
|
||||
const categoryOptions = useMemo<CategoryOption[]>(() => {
|
||||
const seen = new Set<string>();
|
||||
|
|
@ -210,13 +240,24 @@ export default function SkillsDialog({ open, onOpenChange, agentId }: SkillsDial
|
|||
</div>
|
||||
|
||||
<div className="flex-1 overflow-y-auto px-6 py-4">
|
||||
{isSkillsError && (
|
||||
<div
|
||||
role="alert"
|
||||
className="mb-3 flex items-center justify-between gap-3 rounded-xl border border-border-medium px-3 py-2 text-sm text-text-secondary"
|
||||
>
|
||||
<span>{localize('com_ui_skills_load_error')}</span>
|
||||
<Button type="button" variant="outline" onClick={handleRetrySkills}>
|
||||
{localize('com_ui_retry')}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
<MarketplaceCatalog
|
||||
items={filtered}
|
||||
selectedIds={selectedIds}
|
||||
onToggle={handleToggle}
|
||||
onConfigure={setDetailItem}
|
||||
view={view}
|
||||
isLoadingSkills={isLoadingSkills}
|
||||
isLoadingSkills={isLoadingSkills || isFetchingNextPage}
|
||||
skillsInView={view !== 'mine'}
|
||||
favoriteKeys={favoriteKeys}
|
||||
onToggleFavorite={toggleFavorite}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,249 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom/extend-expect';
|
||||
import { fireEvent, render, screen } from '@testing-library/react';
|
||||
import type { TSkillSummary } from 'librechat-data-provider';
|
||||
import SkillsDialog from '../SkillsDialog';
|
||||
|
||||
const mockFetchNextPage = jest.fn();
|
||||
const mockRefetch = jest.fn();
|
||||
const mockUseListSkillsQuery = jest.fn();
|
||||
const mockUseSkillsInfiniteQuery = jest.fn();
|
||||
|
||||
let mockInfiniteResult: {
|
||||
data?: { pages: Array<{ skills: TSkillSummary[] }> };
|
||||
isLoading: boolean;
|
||||
isError: boolean;
|
||||
fetchNextPage: jest.Mock;
|
||||
refetch: jest.Mock;
|
||||
hasNextPage: boolean;
|
||||
isFetchingNextPage: boolean;
|
||||
};
|
||||
|
||||
jest.mock('react-hook-form', () => ({
|
||||
useFormContext: () => ({
|
||||
control: {},
|
||||
getValues: (name: string) => (name === 'skills' ? [] : false),
|
||||
setValue: jest.fn(),
|
||||
}),
|
||||
useWatch: () => [],
|
||||
}));
|
||||
|
||||
jest.mock('~/hooks', () => ({
|
||||
useLocalize: () => (key: string) => {
|
||||
const values: Record<string, string> = {
|
||||
com_ui_skills: 'Skills',
|
||||
com_ui_search_skills: 'Search skills...',
|
||||
com_ui_skills_dialog_description: 'Browse and add skills to your agent.',
|
||||
com_ui_skills_load_error: 'Failed to load skills',
|
||||
com_ui_retry: 'Retry',
|
||||
};
|
||||
return values[key] ?? key;
|
||||
},
|
||||
useHasAccess: () => true,
|
||||
useAuthContext: () => ({ user: { id: 'user-1' } }),
|
||||
useToolFavorites: () => ({
|
||||
favoriteKeys: new Set<string>(),
|
||||
toggle: jest.fn(),
|
||||
}),
|
||||
}));
|
||||
|
||||
jest.mock('~/data-provider', () => ({
|
||||
useListSkillsQuery: (...args: unknown[]) => mockUseListSkillsQuery(...args),
|
||||
useSkillsInfiniteQuery: (...args: unknown[]) => mockUseSkillsInfiniteQuery(...args),
|
||||
}));
|
||||
|
||||
jest.mock('~/components/Prompts', () => ({
|
||||
CategoryIcon: () => <span aria-hidden="true" />,
|
||||
}));
|
||||
|
||||
jest.mock('~/components/Skills/dialogs', () => ({
|
||||
CreateSkillDialog: () => null,
|
||||
}));
|
||||
|
||||
jest.mock('../CategoryFilter', () => ({
|
||||
__esModule: true,
|
||||
default: () => null,
|
||||
}));
|
||||
|
||||
jest.mock('../ItemDialog/ItemDialog', () => ({
|
||||
__esModule: true,
|
||||
default: () => null,
|
||||
}));
|
||||
|
||||
jest.mock('../MarketplaceCatalog', () => ({
|
||||
__esModule: true,
|
||||
default: ({
|
||||
items,
|
||||
ariaLabel,
|
||||
}: {
|
||||
items: Array<{ id: string; name: string }>;
|
||||
ariaLabel: string;
|
||||
}) => (
|
||||
<ul aria-label={ariaLabel}>
|
||||
{items.map((item) => (
|
||||
<li key={item.id}>{item.name}</li>
|
||||
))}
|
||||
</ul>
|
||||
),
|
||||
}));
|
||||
|
||||
jest.mock('@librechat/client', () => ({
|
||||
Button: ({ children, ...props }: React.ComponentProps<'button'>) => (
|
||||
<button type="button" {...props}>
|
||||
{children}
|
||||
</button>
|
||||
),
|
||||
Input: (props: React.ComponentProps<'input'>) => <input {...props} />,
|
||||
Radio: () => null,
|
||||
OGDialog: ({ open, children }: { open: boolean; children: React.ReactNode }) =>
|
||||
open ? <div>{children}</div> : null,
|
||||
OGDialogContent: ({ children }: { children: React.ReactNode }) => (
|
||||
<div role="dialog">{children}</div>
|
||||
),
|
||||
OGDialogTitle: ({ children }: { children: React.ReactNode }) => <h2>{children}</h2>,
|
||||
OGDialogDescription: ({ children }: { children: React.ReactNode }) => <p>{children}</p>,
|
||||
}));
|
||||
|
||||
function makeSkill(id: string, name: string): TSkillSummary {
|
||||
return {
|
||||
_id: id,
|
||||
name,
|
||||
description: `${name} description`,
|
||||
author: 'user-1',
|
||||
authorName: 'E2E User',
|
||||
version: 1,
|
||||
source: 'inline',
|
||||
fileCount: 0,
|
||||
createdAt: '2026-07-24T00:00:00.000Z',
|
||||
updatedAt: '2026-07-24T00:00:00.000Z',
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
mockInfiniteResult = {
|
||||
data: { pages: [{ skills: [makeSkill('first', 'first-skill')] }] },
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
fetchNextPage: mockFetchNextPage,
|
||||
refetch: mockRefetch,
|
||||
hasNextPage: true,
|
||||
isFetchingNextPage: false,
|
||||
};
|
||||
mockFetchNextPage.mockReset();
|
||||
mockRefetch.mockReset();
|
||||
mockUseListSkillsQuery.mockReset();
|
||||
mockUseListSkillsQuery.mockReturnValue({
|
||||
data: { skills: [makeSkill('first', 'first-skill')] },
|
||||
isLoading: false,
|
||||
});
|
||||
mockUseSkillsInfiniteQuery.mockReset();
|
||||
mockUseSkillsInfiniteQuery.mockImplementation(() => mockInfiniteResult);
|
||||
});
|
||||
|
||||
describe('SkillsDialog pagination', () => {
|
||||
test('auto-fetches every page and stops after the final page', () => {
|
||||
const { rerender } = render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
|
||||
expect(mockUseSkillsInfiniteQuery).toHaveBeenCalledWith({ limit: 100 }, { enabled: true });
|
||||
expect(mockFetchNextPage).toHaveBeenCalledTimes(1);
|
||||
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
isFetchingNextPage: true,
|
||||
};
|
||||
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
|
||||
expect(mockFetchNextPage).toHaveBeenCalledTimes(1);
|
||||
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
data: {
|
||||
pages: [
|
||||
{ skills: [makeSkill('first', 'first-skill')] },
|
||||
{ skills: [makeSkill('second', 'second-skill')] },
|
||||
],
|
||||
},
|
||||
hasNextPage: true,
|
||||
isFetchingNextPage: false,
|
||||
};
|
||||
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
|
||||
expect(mockFetchNextPage).toHaveBeenCalledTimes(2);
|
||||
expect(screen.getByText('second-skill')).toBeInTheDocument();
|
||||
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
isFetchingNextPage: true,
|
||||
};
|
||||
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
|
||||
expect(mockFetchNextPage).toHaveBeenCalledTimes(2);
|
||||
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
data: {
|
||||
pages: [
|
||||
{ skills: [makeSkill('first', 'first-skill')] },
|
||||
{ skills: [makeSkill('second', 'second-skill')] },
|
||||
{ skills: [makeSkill('third', 'third-skill')] },
|
||||
],
|
||||
},
|
||||
hasNextPage: false,
|
||||
isFetchingNextPage: false,
|
||||
};
|
||||
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
|
||||
expect(screen.getByText('third-skill')).toBeInTheDocument();
|
||||
expect(mockFetchNextPage).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test('flattens and de-duplicates every loaded page for discovery', () => {
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
data: {
|
||||
pages: [
|
||||
{ skills: [makeSkill('first', 'first-skill')] },
|
||||
{
|
||||
skills: [makeSkill('first', 'first-skill'), makeSkill('off-page', 'off-page-skill')],
|
||||
},
|
||||
],
|
||||
},
|
||||
hasNextPage: false,
|
||||
};
|
||||
|
||||
render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
|
||||
expect(screen.getAllByText('first-skill')).toHaveLength(1);
|
||||
expect(screen.getByText('off-page-skill')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
test('shows a recoverable pagination error and retries on request', () => {
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
isError: true,
|
||||
};
|
||||
render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
expect(mockFetchNextPage).not.toHaveBeenCalled();
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('Failed to load skills');
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Retry' }));
|
||||
expect(mockRefetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('resumes pagination after an external refetch clears the error', () => {
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
isError: true,
|
||||
};
|
||||
const { rerender } = render(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
expect(mockFetchNextPage).not.toHaveBeenCalled();
|
||||
|
||||
mockInfiniteResult = {
|
||||
...mockInfiniteResult,
|
||||
isError: false,
|
||||
};
|
||||
rerender(<SkillsDialog open onOpenChange={jest.fn()} agentId="agent-1" />);
|
||||
|
||||
expect(mockFetchNextPage).toHaveBeenCalledTimes(1);
|
||||
expect(screen.queryByRole('alert')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -13,6 +13,7 @@ import {
|
|||
SKILL_DESCRIPTION_MAX_LENGTH,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TSkill } from 'librechat-data-provider';
|
||||
import type { FormEvent } from 'react';
|
||||
import { useCreateSkillMutation } from '~/data-provider';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import { cn } from '~/utils';
|
||||
|
|
@ -98,13 +99,23 @@ export default function CreateSkillDialog({
|
|||
reset();
|
||||
};
|
||||
|
||||
const handleFormSubmit = (event: FormEvent<HTMLFormElement>) => {
|
||||
/**
|
||||
* This dialog can be portaled from inside Agent Builder's form. React
|
||||
* events still bubble through the component tree across a portal, so an
|
||||
* unguarded submit also submits (and prematurely creates) the agent.
|
||||
*/
|
||||
event.stopPropagation();
|
||||
return handleSubmit(onSubmit)(event);
|
||||
};
|
||||
|
||||
const submitDisabled = !isValid || isSubmitting || createSkill.isLoading;
|
||||
|
||||
return (
|
||||
<OGDialog open={isOpen} onOpenChange={setIsOpen}>
|
||||
<OGDialogContent className="w-11/12 max-w-5xl overflow-hidden" showCloseButton={false}>
|
||||
<form
|
||||
onSubmit={handleSubmit(onSubmit)}
|
||||
onSubmit={handleFormSubmit}
|
||||
className="flex max-h-[80vh] min-w-0 flex-col gap-3 overflow-hidden p-1 sm:gap-4 sm:p-2"
|
||||
>
|
||||
<h2 className="text-lg font-bold text-text-primary">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import type { FormEvent, ReactNode } from 'react';
|
||||
import CreateSkillDialog from '../CreateSkillDialog';
|
||||
|
||||
const mockMutate = jest.fn();
|
||||
const mockNavigate = jest.fn();
|
||||
const mockSetIsOpen = jest.fn();
|
||||
const mockShowToast = jest.fn();
|
||||
|
||||
jest.mock('react-router-dom', () => ({
|
||||
...jest.requireActual('react-router-dom'),
|
||||
useNavigate: () => mockNavigate,
|
||||
}));
|
||||
|
||||
jest.mock(
|
||||
'@librechat/client',
|
||||
() => {
|
||||
const React = jest.requireActual<typeof import('react')>('react');
|
||||
const ReactDOM = jest.requireActual<typeof import('react-dom')>('react-dom');
|
||||
return {
|
||||
Button: ({
|
||||
children,
|
||||
variant: _variant,
|
||||
...props
|
||||
}: React.ButtonHTMLAttributes<HTMLButtonElement> & { variant?: string }) =>
|
||||
React.createElement('button', props, children),
|
||||
OGDialog: ({ open, children }: { open: boolean; children: ReactNode }) =>
|
||||
open ? React.createElement(React.Fragment, null, children) : null,
|
||||
OGDialogContent: ({ children }: { children: ReactNode }) =>
|
||||
ReactDOM.createPortal(React.createElement('div', null, children), globalThis.document.body),
|
||||
TextareaAutosize: React.forwardRef<
|
||||
HTMLTextAreaElement,
|
||||
React.TextareaHTMLAttributes<HTMLTextAreaElement> & {
|
||||
minRows?: number;
|
||||
maxRows?: number;
|
||||
}
|
||||
>(({ minRows: _minRows, maxRows: _maxRows, ...props }, ref) =>
|
||||
React.createElement('textarea', { ...props, ref }),
|
||||
),
|
||||
useToastContext: () => ({
|
||||
showToast: mockShowToast,
|
||||
}),
|
||||
};
|
||||
},
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
jest.mock('~/data-provider', () => ({
|
||||
useCreateSkillMutation: () => ({
|
||||
mutate: mockMutate,
|
||||
isLoading: false,
|
||||
}),
|
||||
}));
|
||||
|
||||
jest.mock('~/hooks', () => ({
|
||||
useLocalize: () => (key: string) => {
|
||||
const translations: Record<string, string> = {
|
||||
com_ui_name: 'Name',
|
||||
com_ui_description: 'Description',
|
||||
com_ui_skill_instructions: 'Instructions',
|
||||
com_ui_cancel: 'Cancel',
|
||||
com_ui_create: 'Create',
|
||||
};
|
||||
return translations[key] ?? key;
|
||||
},
|
||||
}));
|
||||
|
||||
jest.mock('~/utils', () => ({
|
||||
cn: (...classes: Array<string | false | null | undefined>) => classes.filter(Boolean).join(' '),
|
||||
}));
|
||||
|
||||
describe('CreateSkillDialog', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('does not submit an ancestor form when rendered through a portal', async () => {
|
||||
const handleAgentSubmit = jest.fn((event: FormEvent<HTMLFormElement>) =>
|
||||
event.preventDefault(),
|
||||
);
|
||||
|
||||
render(
|
||||
<form onSubmit={handleAgentSubmit}>
|
||||
<CreateSkillDialog isOpen={true} setIsOpen={mockSetIsOpen} />
|
||||
</form>,
|
||||
);
|
||||
|
||||
fireEvent.change(screen.getByLabelText('Name'), {
|
||||
target: { value: 'portal-skill' },
|
||||
});
|
||||
fireEvent.change(screen.getByLabelText('Description'), {
|
||||
target: { value: 'A skill created from Agent Builder' },
|
||||
});
|
||||
fireEvent.change(screen.getByLabelText('Instructions'), {
|
||||
target: { value: 'Apply the portal skill.' },
|
||||
});
|
||||
|
||||
const createButton = screen.getByRole('button', { name: 'Create' });
|
||||
await waitFor(() => expect(createButton).toBeEnabled());
|
||||
fireEvent.click(createButton);
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mockMutate).toHaveBeenCalledWith({
|
||||
name: 'portal-skill',
|
||||
description: 'A skill created from Agent Builder',
|
||||
body: 'Apply the portal skill.',
|
||||
}),
|
||||
);
|
||||
expect(handleAgentSubmit).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
142
client/src/data-provider/Skills/__tests__/mutations.spec.tsx
Normal file
142
client/src/data-provider/Skills/__tests__/mutations.spec.tsx
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
import React from 'react';
|
||||
import { QueryKeys } from 'librechat-data-provider';
|
||||
import { act, renderHook, waitFor } from '@testing-library/react';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
import type { TSkill, TSkillListResponse } from 'librechat-data-provider';
|
||||
import type { ReactNode } from 'react';
|
||||
import { useCreateSkillMutation } from '../mutations';
|
||||
import { useSkillsInfiniteQuery } from '../queries';
|
||||
|
||||
const mockCreateSkill = jest.fn();
|
||||
const mockListSkills = jest.fn();
|
||||
|
||||
jest.mock('librechat-data-provider', () => {
|
||||
const actual = jest.requireActual('librechat-data-provider');
|
||||
return {
|
||||
...actual,
|
||||
dataService: {
|
||||
...actual.dataService,
|
||||
createSkill: (...args: unknown[]) => mockCreateSkill(...args),
|
||||
listSkills: (...args: unknown[]) => mockListSkills(...args),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
function makeSkill(id: string, name: string): TSkill {
|
||||
return {
|
||||
_id: id,
|
||||
name,
|
||||
description: `${name} description`,
|
||||
body: `# ${name}`,
|
||||
author: 'user-1',
|
||||
authorName: 'E2E User',
|
||||
version: 1,
|
||||
source: 'inline',
|
||||
fileCount: 0,
|
||||
createdAt: '2026-07-24T00:00:00.000Z',
|
||||
updatedAt: '2026-07-24T00:00:00.000Z',
|
||||
} as TSkill;
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe('skill creation cache updates', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('keeps a created skill when a stale next-page request finishes afterward', async () => {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: { retry: false },
|
||||
mutations: { retry: false },
|
||||
},
|
||||
});
|
||||
const firstSkill = makeSkill('first-id', 'first-skill');
|
||||
const createdSkill = makeSkill('created-id', 'created-skill');
|
||||
const stalePage = deferred<TSkillListResponse>();
|
||||
const firstPage: TSkillListResponse = {
|
||||
skills: [firstSkill],
|
||||
has_more: true,
|
||||
after: 'cursor-1',
|
||||
};
|
||||
const freshPage: TSkillListResponse = {
|
||||
skills: [createdSkill, firstSkill],
|
||||
has_more: false,
|
||||
after: null,
|
||||
};
|
||||
const queryKey = [QueryKeys.skills, 'infinite', '', '', 100];
|
||||
queryClient.setQueryData(queryKey, {
|
||||
pages: [firstPage],
|
||||
pageParams: [undefined],
|
||||
});
|
||||
mockCreateSkill.mockResolvedValue(createdSkill);
|
||||
mockListSkills.mockImplementation(({ cursor }: { cursor?: string }) =>
|
||||
cursor === 'cursor-1' ? stalePage.promise : Promise.resolve(freshPage),
|
||||
);
|
||||
const cancelQueries = jest.spyOn(queryClient, 'cancelQueries');
|
||||
const wrapper = ({ children }: { children: ReactNode }) => (
|
||||
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => ({
|
||||
list: useSkillsInfiniteQuery({ limit: 100 }),
|
||||
create: useCreateSkillMutation(),
|
||||
}),
|
||||
{ wrapper },
|
||||
);
|
||||
|
||||
let staleRequest: Promise<unknown>;
|
||||
act(() => {
|
||||
staleRequest = result.current.list.fetchNextPage();
|
||||
});
|
||||
await waitFor(() =>
|
||||
expect(mockListSkills).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ cursor: 'cursor-1', limit: 100 }),
|
||||
),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
await result.current.create.mutateAsync({
|
||||
name: createdSkill.name,
|
||||
description: createdSkill.description,
|
||||
body: createdSkill.body,
|
||||
});
|
||||
});
|
||||
|
||||
expect(cancelQueries).toHaveBeenCalledWith([QueryKeys.skills]);
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
queryClient
|
||||
.getQueryData<{ pages: TSkillListResponse[] }>(queryKey)
|
||||
?.pages.flatMap((page) => page.skills)
|
||||
.map((skill) => skill._id),
|
||||
).toContain(createdSkill._id),
|
||||
);
|
||||
|
||||
stalePage.resolve({
|
||||
skills: [makeSkill('second-id', 'second-skill')],
|
||||
has_more: false,
|
||||
after: null,
|
||||
});
|
||||
await act(async () => {
|
||||
await Promise.allSettled([staleRequest]);
|
||||
});
|
||||
|
||||
expect(
|
||||
queryClient
|
||||
.getQueryData<{ pages: TSkillListResponse[] }>(queryKey)
|
||||
?.pages.flatMap((page) => page.skills)
|
||||
.map((skill) => skill._id),
|
||||
).toContain(createdSkill._id);
|
||||
|
||||
queryClient.clear();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
import { useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { QueryKeys, dataService } from 'librechat-data-provider';
|
||||
import type { InfiniteData, QueryKey, UseMutationResult } from '@tanstack/react-query';
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import type {
|
||||
TSkill,
|
||||
TSkillFile,
|
||||
|
|
@ -21,6 +20,7 @@ import type {
|
|||
ImportSkillOptions,
|
||||
DeleteSkillFileOptions,
|
||||
} from 'librechat-data-provider';
|
||||
import type { InfiniteData, QueryKey, UseMutationResult } from '@tanstack/react-query';
|
||||
|
||||
function isInfiniteSkillData(
|
||||
data: TSkillListResponse | InfiniteData<TSkillListResponse>,
|
||||
|
|
@ -49,14 +49,21 @@ function addSkillToCachedLists(
|
|||
queryClient.setQueriesData<TSkillCacheEntry>([QueryKeys.skills], (data) => {
|
||||
if (!data) return data;
|
||||
if (isInfiniteSkillData(data)) {
|
||||
const pages = data.pages.map((page) => ({
|
||||
...page,
|
||||
skills: page.skills.filter((existing) => existing._id !== skill._id),
|
||||
}));
|
||||
return {
|
||||
...data,
|
||||
pages: data.pages.map((page, i) =>
|
||||
pages: pages.map((page, i) =>
|
||||
i === 0 ? { ...page, skills: [skill, ...page.skills] } : page,
|
||||
),
|
||||
};
|
||||
}
|
||||
return { ...data, skills: [skill, ...data.skills] };
|
||||
return {
|
||||
...data,
|
||||
skills: [skill, ...data.skills.filter((existing) => existing._id !== skill._id)],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -119,9 +126,16 @@ export const useCreateSkillMutation = (
|
|||
return useMutation({
|
||||
mutationFn: (payload: TCreateSkill) => dataService.createSkill(payload),
|
||||
...rest,
|
||||
onSuccess: (skill, variables, context) => {
|
||||
onSuccess: async (skill, variables, context) => {
|
||||
/**
|
||||
* An infinite skill query may have captured its pre-create pages before
|
||||
* this mutation completed. Cancel it before the cache write so a late
|
||||
* cursor response cannot replace the newly-created row.
|
||||
*/
|
||||
await queryClient.cancelQueries([QueryKeys.skills]);
|
||||
queryClient.setQueryData<TSkill>([QueryKeys.skill, skill._id], skill);
|
||||
addSkillToCachedLists(queryClient, skill);
|
||||
void queryClient.invalidateQueries([QueryKeys.skills]);
|
||||
if (onSuccess) onSuccess(skill, variables, context);
|
||||
},
|
||||
});
|
||||
|
|
@ -139,9 +153,11 @@ export const useImportSkillMutation = (
|
|||
return useMutation({
|
||||
mutationFn: (formData: FormData) => dataService.importSkill(formData),
|
||||
...rest,
|
||||
onSuccess: (skill, variables, context) => {
|
||||
onSuccess: async (skill, variables, context) => {
|
||||
await queryClient.cancelQueries([QueryKeys.skills]);
|
||||
queryClient.setQueryData<TSkill>([QueryKeys.skill, skill._id], skill);
|
||||
addSkillToCachedLists(queryClient, skill);
|
||||
void queryClient.invalidateQueries([QueryKeys.skills]);
|
||||
if (onSuccess) onSuccess(skill, variables, context);
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
import { resolveSkillDefaultActive } from '../useSkillActiveState';
|
||||
|
||||
describe('resolveSkillDefaultActive', () => {
|
||||
test('activates deployment skills for every user by default', () => {
|
||||
expect(
|
||||
resolveSkillDefaultActive(
|
||||
{ author: 'deployment-registry', source: 'deployment' },
|
||||
'user-1',
|
||||
false,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test('activates owned inline skills by default', () => {
|
||||
expect(resolveSkillDefaultActive({ author: 'user-1', source: 'inline' }, 'user-1', false)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('uses the shared-skill configuration for non-owned inline skills', () => {
|
||||
const skill = { author: 'user-2', source: 'inline' } as const;
|
||||
expect(resolveSkillDefaultActive(skill, 'user-1', false)).toBe(false);
|
||||
expect(resolveSkillDefaultActive(skill, 'user-1', true)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import { useCallback, useEffect, useMemo } from 'react';
|
||||
import { useQueryClient } from '@tanstack/react-query';
|
||||
import { QueryKeys } from 'librechat-data-provider';
|
||||
import { useToastContext } from '@librechat/client';
|
||||
import type { TSkillStatesResponse } from 'librechat-data-provider';
|
||||
import { useQueryClient } from '@tanstack/react-query';
|
||||
import type { TSkillStatesResponse, TSkillSummary } from 'librechat-data-provider';
|
||||
import {
|
||||
useGetSkillStatesQuery,
|
||||
useUpdateSkillStatesMutation,
|
||||
|
|
@ -12,6 +12,7 @@ import { useAuthContext, useLocalize } from '~/hooks';
|
|||
import { logger } from '~/utils';
|
||||
|
||||
const EMPTY_STATES: TSkillStatesResponse = {};
|
||||
type SkillActiveTarget = Pick<TSkillSummary, '_id' | 'author' | 'source'>;
|
||||
|
||||
interface WriteQueue {
|
||||
pending: TSkillStatesResponse | null;
|
||||
|
|
@ -65,19 +66,28 @@ function getWriteQueue(userId: string): WriteQueue {
|
|||
/**
|
||||
* Resolves the default active state for a skill the user has never toggled.
|
||||
*
|
||||
* - Deployment skills default to **active** for every user.
|
||||
* - Owned skills (author === currentUser) default to **active**.
|
||||
* - Shared skills default to the `defaultActiveOnShare` config value (default `false`).
|
||||
*/
|
||||
function resolveDefault(author: string, userId: string, defaultActiveOnShare: boolean): boolean {
|
||||
return author === userId ? true : defaultActiveOnShare;
|
||||
export function resolveSkillDefaultActive(
|
||||
skill: Pick<SkillActiveTarget, 'author' | 'source'>,
|
||||
userId: string,
|
||||
defaultActiveOnShare: boolean,
|
||||
): boolean {
|
||||
if (skill.source === 'deployment' || skill.author === userId) {
|
||||
return true;
|
||||
}
|
||||
return defaultActiveOnShare;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for managing per-user skill active/inactive state.
|
||||
*
|
||||
* The `skillStates` map stores explicit overrides (`{ [skillId]: boolean }`).
|
||||
* Skills absent from the map use the ownership-based default: owned -> active,
|
||||
* shared -> `defaultActiveOnShare` from the interface config. Toggles that
|
||||
* Skills absent from the map use the source/ownership default: deployment and
|
||||
* owned skills -> active, shared skills -> `defaultActiveOnShare` from the
|
||||
* interface config. Toggles that
|
||||
* land on the resolved default remove the key from the map rather than
|
||||
* persisting a redundant entry, keeping `skillStates` strictly an exceptions
|
||||
* list (otherwise rapid round-trip toggles would exhaust the 200-entry cap).
|
||||
|
|
@ -165,18 +175,18 @@ export default function useSkillActiveState() {
|
|||
}, [userId, updateMutation, showToast, localize]);
|
||||
|
||||
const isActive = useCallback(
|
||||
(skill: { _id: string; author: string }): boolean => {
|
||||
(skill: SkillActiveTarget): boolean => {
|
||||
const override = skillStates[skill._id];
|
||||
if (override !== undefined) {
|
||||
return override;
|
||||
}
|
||||
return resolveDefault(skill.author, userId, defaultActiveOnShare);
|
||||
return resolveSkillDefaultActive(skill, userId, defaultActiveOnShare);
|
||||
},
|
||||
[skillStates, userId, defaultActiveOnShare],
|
||||
);
|
||||
|
||||
const toggle = useCallback(
|
||||
(skill: { _id: string; author: string }) => {
|
||||
(skill: SkillActiveTarget) => {
|
||||
if (!canToggle || !userId) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -184,7 +194,7 @@ export default function useSkillActiveState() {
|
|||
const cached =
|
||||
queryClient.getQueryData<TSkillStatesResponse>([QueryKeys.skillStates]) ?? EMPTY_STATES;
|
||||
const baseline = queue.pending ?? cached;
|
||||
const defaultValue = resolveDefault(skill.author, userId, defaultActiveOnShare);
|
||||
const defaultValue = resolveSkillDefaultActive(skill, userId, defaultActiveOnShare);
|
||||
const override = baseline[skill._id];
|
||||
const currentActive = override !== undefined ? override : defaultValue;
|
||||
const nextValue = !currentActive;
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@ modelSpecs:
|
|||
model: 'mock-model-a'
|
||||
skills:
|
||||
- 'e2e-model-spec-allowed'
|
||||
- 'e2e-deployment-skill'
|
||||
- 'e2e-model-spec-missing'
|
||||
- 'e2e-model-spec-inaccessible'
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;
|
|||
|
||||
const CREATE_SKILL_MARKER = 'E2E_CREATE_SKILL:';
|
||||
const EDIT_SKILL_MARKER = 'E2E_EDIT_SKILL:';
|
||||
const ASSERT_MODEL_SPEC_SKILLS_MARKER = 'E2E_ASSERT_MODEL_SPEC_SKILLS';
|
||||
const ASSERT_SKILLS_MARKER = 'E2E_ASSERT_SKILLS:';
|
||||
const ASSERT_MANUAL_SKILL_MARKER = 'E2E_ASSERT_MANUAL_SKILL:';
|
||||
const INVOKE_SKILL_MARKER = 'E2E_INVOKE_SKILL:';
|
||||
const ASSERT_PROVIDER_FILE_MARKER = 'E2E_ASSERT_PROVIDER_FILE:';
|
||||
const ASSERT_AGENT_CONTEXT_MARKER = 'E2E_ASSERT_AGENT_CONTEXT:';
|
||||
const ASSERT_QUOTE_MARKER = 'E2E_ASSERT_QUOTE:';
|
||||
|
|
@ -34,7 +36,9 @@ const BACKGROUND_DISPATCH_MARKER = 'E2E_BACKGROUND_DISPATCH:';
|
|||
const BACKGROUND_COLLECT_MARKER = 'E2E_BACKGROUND_COLLECT:';
|
||||
const CREATE_FILE_AUTHORING_FINAL_TEXT = 'E2E file authoring complete';
|
||||
const EDIT_FILE_AUTHORING_FINAL_TEXT = 'E2E file edit complete';
|
||||
const MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed';
|
||||
const SKILL_ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
|
||||
const MANUAL_SKILL_ASSERTION_FINAL_TEXT = 'E2E manual skill assertion passed';
|
||||
const SKILL_TOOL_ASSERTION_FINAL_TEXT = 'E2E skill tool assertion passed';
|
||||
const PROVIDER_FILE_ASSERTION_FINAL_TEXT = 'E2E provider file assertion passed';
|
||||
const AGENT_CONTEXT_ASSERTION_FINAL_TEXT = 'E2E agent context assertion passed';
|
||||
const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed';
|
||||
|
|
@ -55,9 +59,9 @@ const CHECK_BACKGROUND_TASK_TOOL_NAME = 'check_background_task';
|
|||
const BACKGROUND_DISPATCH_TOOL_CALL_ID = 'call_e2e_background_dispatch';
|
||||
const BACKGROUND_COLLECT_TOOL_CALL_ID = 'call_e2e_background_collect';
|
||||
const MODEL_SPEC_ACCESSIBLE_SKILL = 'e2e-model-spec-allowed';
|
||||
const MODEL_SPEC_MISSING_SKILL = 'e2e-model-spec-missing';
|
||||
const MODEL_SPEC_INACCESSIBLE_SKILL = 'e2e-model-spec-inaccessible';
|
||||
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
|
||||
const ALWAYS_APPLY_BODY_MARKER = 'E2E_ALWAYS_APPLY_BODY_MARKER';
|
||||
const DEPLOYMENT_SKILL_BODY_MARKER = 'E2E deployment skill loaded through Playwright';
|
||||
const SKILL_DESCRIPTION =
|
||||
'Use this skill to verify LibreChat skill file authoring in mock end-to-end tests.';
|
||||
const EDITED_SKILL_DESCRIPTION =
|
||||
|
|
@ -165,13 +169,30 @@ function collectToolNames(agents) {
|
|||
return names;
|
||||
}
|
||||
|
||||
function collectAdditionalInstructions(agents) {
|
||||
return (agents ?? [])
|
||||
.map((agent) =>
|
||||
typeof agent?.additional_instructions === 'string' ? agent.additional_instructions : '',
|
||||
)
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
async function getStreamAgentView({ graph, messages, options, runManager }) {
|
||||
let agentId = runManager?.metadata?.agentId ?? options?.metadata?.agentId;
|
||||
let agentContext;
|
||||
if (typeof agentId === 'string') {
|
||||
agentContext = graph?.agentContexts?.get(agentId);
|
||||
} else if (graph?.agentContexts?.size === 1) {
|
||||
[agentId, agentContext] = graph.agentContexts.entries().next().value;
|
||||
}
|
||||
/**
|
||||
* Graph.attemptInvoke intentionally sends test override models the pruned
|
||||
* messages directly, bypassing the production model's systemRunnable pipe.
|
||||
* Apply that agent's runnable here so assertions inspect the same complete
|
||||
* prompt (system catalog plus messages) that a real provider receives.
|
||||
*/
|
||||
const systemRunnable = agentContext?.systemRunnable;
|
||||
const promptMessages =
|
||||
systemRunnable && typeof systemRunnable.invoke === 'function'
|
||||
? await systemRunnable.invoke(messages)
|
||||
: messages;
|
||||
return {
|
||||
agentId,
|
||||
messages: promptMessages,
|
||||
toolNames: collectToolNames(agentContext ? [agentContext] : []),
|
||||
};
|
||||
}
|
||||
|
||||
function collectPromptText(value, parts = []) {
|
||||
|
|
@ -443,7 +464,7 @@ class UsageEmittingFakeChatModel extends FakeChatModel {
|
|||
|
||||
async *_streamResponseChunks(messages, options, runManager) {
|
||||
let outputChars = 0;
|
||||
const dynamicResponse = this.resolveOnStream?.(messages);
|
||||
const dynamicResponse = await this.resolveOnStream?.(messages, options, runManager);
|
||||
const chunkStream = dynamicResponse
|
||||
? this.streamDynamicResponseChunks({
|
||||
responses: dynamicResponse.responses,
|
||||
|
|
@ -502,45 +523,208 @@ function overrideModel({ graph, responses, sleep, toolCalls, thrownError, resolv
|
|||
});
|
||||
}
|
||||
|
||||
function modelSpecSkillAssertionResponses({ agents, messages, toolNames }) {
|
||||
const failures = [];
|
||||
const additionalInstructions = collectAdditionalInstructions(agents);
|
||||
const skillPrimeMessages = collectSkillPrimeMessages(messages);
|
||||
const alwaysApplyPrime = skillPrimeMessages.find(
|
||||
(message) => message.name === MODEL_SPEC_ACCESSIBLE_SKILL && message.trigger === 'always-apply',
|
||||
);
|
||||
function parseSkillAssertion(text, agentId) {
|
||||
const markerValue = getMarkerValue(text, ASSERT_SKILLS_MARKER);
|
||||
const sections = markerValue
|
||||
.split(';')
|
||||
.map((section) => section.trim())
|
||||
.filter(Boolean);
|
||||
const isAgentScoped = sections.some((section) => section.includes('='));
|
||||
let entriesValue = markerValue;
|
||||
if (isAgentScoped) {
|
||||
// Parallel agents receive a per-run `____N` suffix while the request and
|
||||
// persisted Agent Builder state retain the stable agent id.
|
||||
const persistedAgentId =
|
||||
typeof agentId === 'string' ? agentId.replace(/____\d+$/, '') : agentId;
|
||||
const prefixes = [`${agentId}=`, `${persistedAgentId}=`];
|
||||
const scopedSection = sections.find((section) =>
|
||||
prefixes.some((prefix) => section.startsWith(prefix)),
|
||||
);
|
||||
if (!scopedSection) {
|
||||
return {
|
||||
required: [],
|
||||
requiredBodies: [],
|
||||
forbidden: [],
|
||||
error: `no skill assertion was configured for agent ${agentId ?? 'unknown'}`,
|
||||
};
|
||||
}
|
||||
const prefix = prefixes.find((candidate) => scopedSection.startsWith(candidate));
|
||||
entriesValue = scopedSection.slice(prefix.length);
|
||||
}
|
||||
|
||||
if (!toolNames.has(SKILL_TOOL_NAME)) {
|
||||
const entries = entriesValue
|
||||
.split(',')
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
return entries.reduce(
|
||||
(assertion, entry) => {
|
||||
if (entry.startsWith('!')) {
|
||||
const name = entry.slice(1);
|
||||
if (name) {
|
||||
assertion.forbidden.push(name);
|
||||
}
|
||||
return assertion;
|
||||
}
|
||||
if (entry.startsWith('*')) {
|
||||
const name = entry.slice(1);
|
||||
if (name) {
|
||||
assertion.required.push(name);
|
||||
assertion.requiredBodies.push(name);
|
||||
}
|
||||
return assertion;
|
||||
}
|
||||
assertion.required.push(entry);
|
||||
return assertion;
|
||||
},
|
||||
{ required: [], requiredBodies: [], forbidden: [], error: undefined },
|
||||
);
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function promptHasSkillCatalogEntry(promptText, skillName) {
|
||||
if (!promptText.includes('## Available Skills')) {
|
||||
return false;
|
||||
}
|
||||
return new RegExp(`(?:^|\\n)- ${escapeRegExp(skillName)}(?::|\\s*(?:\\n|$))`, 'm').test(
|
||||
promptText,
|
||||
);
|
||||
}
|
||||
|
||||
function expectedSkillBodyMarker(skillName) {
|
||||
if (skillName === MODEL_SPEC_ACCESSIBLE_SKILL) {
|
||||
return ALWAYS_APPLY_BODY_MARKER;
|
||||
}
|
||||
if (skillName === DEPLOYMENT_SKILL_NAME) {
|
||||
return DEPLOYMENT_SKILL_BODY_MARKER;
|
||||
}
|
||||
return `# ${skillName}`;
|
||||
}
|
||||
|
||||
function skillAssertionResponses({ messages, assertion, toolNames }) {
|
||||
const failures = [];
|
||||
if (assertion.error) {
|
||||
failures.push(assertion.error);
|
||||
}
|
||||
const promptText = collectPromptText(messages).join('\n');
|
||||
const skillPrimeMessages = collectSkillPrimeMessages(messages);
|
||||
|
||||
if (assertion.required.length > 0 && !toolNames.has(SKILL_TOOL_NAME)) {
|
||||
failures.push(`${SKILL_TOOL_NAME} tool was not advertised`);
|
||||
}
|
||||
if (!additionalInstructions.includes(MODEL_SPEC_ACCESSIBLE_SKILL)) {
|
||||
failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} was not present in the model-visible catalog`);
|
||||
if (assertion.required.length === 0 && toolNames.has(SKILL_TOOL_NAME)) {
|
||||
failures.push(`${SKILL_TOOL_NAME} tool was unexpectedly advertised`);
|
||||
}
|
||||
if (additionalInstructions.includes(MODEL_SPEC_MISSING_SKILL)) {
|
||||
failures.push(`${MODEL_SPEC_MISSING_SKILL} leaked into the model-visible catalog`);
|
||||
for (const name of assertion.required) {
|
||||
if (!promptHasSkillCatalogEntry(promptText, name)) {
|
||||
failures.push(`${name} was not present in the model-visible catalog`);
|
||||
}
|
||||
}
|
||||
if (additionalInstructions.includes(MODEL_SPEC_INACCESSIBLE_SKILL)) {
|
||||
failures.push(`${MODEL_SPEC_INACCESSIBLE_SKILL} leaked into the model-visible catalog`);
|
||||
for (const name of assertion.requiredBodies) {
|
||||
const expectedMarker = expectedSkillBodyMarker(name);
|
||||
const taggedBody = skillPrimeMessages.find((message) => message.name === name);
|
||||
if (!taggedBody?.content.includes(expectedMarker)) {
|
||||
failures.push(`${name} body was missing its expected marker "${expectedMarker}"`);
|
||||
}
|
||||
}
|
||||
if (!alwaysApplyPrime) {
|
||||
failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} was not always-apply primed`);
|
||||
} else if (!alwaysApplyPrime.content.includes(ALWAYS_APPLY_BODY_MARKER)) {
|
||||
failures.push(`${MODEL_SPEC_ACCESSIBLE_SKILL} always-apply body was missing its marker`);
|
||||
for (const name of assertion.forbidden) {
|
||||
if (promptHasSkillCatalogEntry(promptText, name)) {
|
||||
failures.push(`${name} leaked into the model-visible catalog`);
|
||||
}
|
||||
if (skillPrimeMessages.some((message) => message.name === name)) {
|
||||
failures.push(`${name} was unexpectedly primed`);
|
||||
}
|
||||
if (name === MODEL_SPEC_ACCESSIBLE_SKILL && promptText.includes(ALWAYS_APPLY_BODY_MARKER)) {
|
||||
failures.push(`${name} always-apply body marker leaked into the model prompt`);
|
||||
}
|
||||
if (name === DEPLOYMENT_SKILL_NAME && promptText.includes(DEPLOYMENT_SKILL_BODY_MARKER)) {
|
||||
failures.push(`${name} always-apply body marker leaked into the model prompt`);
|
||||
}
|
||||
}
|
||||
if (skillPrimeMessages.some((message) => message.name === MODEL_SPEC_MISSING_SKILL)) {
|
||||
failures.push(`${MODEL_SPEC_MISSING_SKILL} was unexpectedly primed`);
|
||||
}
|
||||
if (skillPrimeMessages.some((message) => message.name === MODEL_SPEC_INACCESSIBLE_SKILL)) {
|
||||
failures.push(`${MODEL_SPEC_INACCESSIBLE_SKILL} was unexpectedly primed`);
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
return {
|
||||
responses: [`E2E model spec skill assertion failed: ${failures.join('; ')}`],
|
||||
responses: [`E2E skill assertion failed: ${failures.join('; ')}`],
|
||||
};
|
||||
}
|
||||
return {
|
||||
responses: [`${MODEL_SPEC_SKILL_ASSERTION_FINAL_TEXT}: ${MODEL_SPEC_ACCESSIBLE_SKILL}`],
|
||||
responses: [
|
||||
`${SKILL_ASSERTION_FINAL_TEXT}: ${
|
||||
assertion.required.length > 0 ? assertion.required.join(', ') : 'none'
|
||||
}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function manualSkillAssertionResponses({ messages, skillName }) {
|
||||
const taggedPrime = collectSkillPrimeMessages(messages).find(
|
||||
(message) => message.name === skillName && message.trigger === 'manual',
|
||||
);
|
||||
if (!taggedPrime) {
|
||||
return {
|
||||
responses: [`E2E manual skill assertion failed: ${skillName} was not manually primed`],
|
||||
};
|
||||
}
|
||||
if (!taggedPrime.content.includes(`# ${skillName}`)) {
|
||||
return {
|
||||
responses: [`E2E manual skill assertion failed: ${skillName} body was missing`],
|
||||
};
|
||||
}
|
||||
return {
|
||||
responses: [`${MANUAL_SKILL_ASSERTION_FINAL_TEXT}: ${skillName}`],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Exercises the real event-driven `skill` handler. The first model call emits
|
||||
* the tool request; the second verifies both the visible tool result and the
|
||||
* body-bearing meta HumanMessage that ToolNode reinjected for the model.
|
||||
*/
|
||||
function skillToolInvocationResponses({ skillName, toolNames }) {
|
||||
if (!toolNames.has(SKILL_TOOL_NAME)) {
|
||||
return {
|
||||
responses: [`E2E skill tool assertion failed: ${SKILL_TOOL_NAME} was not advertised`],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
responses: ['', ''],
|
||||
toolCalls: [
|
||||
{
|
||||
id: `call_e2e_skill_${skillName}`,
|
||||
name: SKILL_TOOL_NAME,
|
||||
args: { skillName },
|
||||
type: 'tool_call',
|
||||
},
|
||||
],
|
||||
resolveOnStream: (streamMessages) => {
|
||||
const toolResult = findLastToolMessageText(
|
||||
streamMessages,
|
||||
`Skill "${skillName}" loaded. Follow the instructions below.`,
|
||||
);
|
||||
if (!toolResult) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const expectedMarker = expectedSkillBodyMarker(skillName);
|
||||
const modelInvokedPrime = collectSkillPrimeMessages(streamMessages).find(
|
||||
(message) =>
|
||||
message.name === skillName &&
|
||||
message.trigger == null &&
|
||||
message.content.includes(expectedMarker),
|
||||
);
|
||||
if (!modelInvokedPrime) {
|
||||
return {
|
||||
responses: [
|
||||
`E2E skill tool assertion failed: ${skillName} body was not reinjected with marker "${expectedMarker}"`,
|
||||
],
|
||||
};
|
||||
}
|
||||
return {
|
||||
responses: [`${SKILL_TOOL_ASSERTION_FINAL_TEXT}: ${skillName}`],
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -740,7 +924,7 @@ function backgroundCollectResponses(messages, toolNames) {
|
|||
};
|
||||
}
|
||||
|
||||
function resolveResponses({ agents, messages, text, toolNames }) {
|
||||
function resolveResponses({ graph, messages, text, toolNames }) {
|
||||
const reply = replyResponses(text);
|
||||
if (reply) {
|
||||
return reply;
|
||||
|
|
@ -769,8 +953,50 @@ function resolveResponses({ agents, messages, text, toolNames }) {
|
|||
return quoteAssertion;
|
||||
}
|
||||
|
||||
if (text.includes(ASSERT_MODEL_SPEC_SKILLS_MARKER)) {
|
||||
return modelSpecSkillAssertionResponses({ agents, messages, toolNames });
|
||||
if (text.includes(ASSERT_SKILLS_MARKER)) {
|
||||
return {
|
||||
responses: [MOCK_REPLY],
|
||||
resolveOnStream: async (streamMessages, streamOptions, runManager) => {
|
||||
const agentView = await getStreamAgentView({
|
||||
graph,
|
||||
messages: streamMessages,
|
||||
options: streamOptions,
|
||||
runManager,
|
||||
});
|
||||
return skillAssertionResponses({
|
||||
messages: agentView.messages,
|
||||
assertion: parseSkillAssertion(text, agentView.agentId),
|
||||
toolNames: agentView.toolNames,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (text.includes(ASSERT_MANUAL_SKILL_MARKER)) {
|
||||
const skillName = getMarkerValue(text, ASSERT_MANUAL_SKILL_MARKER);
|
||||
return {
|
||||
responses: [MOCK_REPLY],
|
||||
resolveOnStream: async (streamMessages, streamOptions, runManager) => {
|
||||
const agentView = await getStreamAgentView({
|
||||
graph,
|
||||
messages: streamMessages,
|
||||
options: streamOptions,
|
||||
runManager,
|
||||
});
|
||||
return manualSkillAssertionResponses({
|
||||
messages: agentView.messages,
|
||||
skillName,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const invokedSkillName = getMarkerValue(text, INVOKE_SKILL_MARKER);
|
||||
if (invokedSkillName) {
|
||||
return skillToolInvocationResponses({
|
||||
skillName: invokedSkillName,
|
||||
toolNames,
|
||||
});
|
||||
}
|
||||
|
||||
const createSkillName = getRequestedSkillName(text, CREATE_SKILL_MARKER);
|
||||
|
|
@ -824,7 +1050,7 @@ module.exports = function fakeModelHook(run, context) {
|
|||
const text = getLatestUserText(context?.messages);
|
||||
const toolNames = collectToolNames(context?.agents);
|
||||
const { responses, sleep, toolCalls, thrownError, resolveOnStream } = resolveResponses({
|
||||
agents: context?.agents,
|
||||
graph,
|
||||
messages: context?.messages,
|
||||
text,
|
||||
toolNames,
|
||||
|
|
|
|||
174
e2e/specs/mock/agent-skills-added.spec.ts
Normal file
174
e2e/specs/mock/agent-skills-added.spec.ts
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Page } from '@playwright/test';
|
||||
import { cleanupAgent, openAgentBuilder, uniqueAgentName } from './agents.helpers';
|
||||
import { NEW_CHAT_PATH, fetchJson, getAccessToken, requestJson, sendMessage } from './helpers';
|
||||
|
||||
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
|
||||
const ASSERTION_MARKER = 'E2E_ASSERT_SKILLS:';
|
||||
const ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
|
||||
|
||||
type SkillSummary = {
|
||||
_id: string;
|
||||
name: string;
|
||||
source?: string;
|
||||
};
|
||||
|
||||
type AgentResponse = {
|
||||
id: string;
|
||||
name?: string | null;
|
||||
skills?: string[];
|
||||
skills_enabled?: boolean;
|
||||
};
|
||||
|
||||
type AgentChatPayload = {
|
||||
addedConvo?: {
|
||||
agent_id?: string;
|
||||
endpoint?: string;
|
||||
};
|
||||
};
|
||||
|
||||
async function getDeploymentSkill(page: Page, token: string): Promise<SkillSummary> {
|
||||
const result = await fetchJson<{ skills?: SkillSummary[] }>(
|
||||
page,
|
||||
`/api/skills?search=${encodeURIComponent(DEPLOYMENT_SKILL_NAME)}&limit=10`,
|
||||
token,
|
||||
);
|
||||
const skill = result.skills?.find((item) => item.name === DEPLOYMENT_SKILL_NAME);
|
||||
expect(skill).toMatchObject({
|
||||
name: DEPLOYMENT_SKILL_NAME,
|
||||
source: 'deployment',
|
||||
});
|
||||
return skill!;
|
||||
}
|
||||
|
||||
async function selectAddedAgent(page: Page, agentName: string) {
|
||||
const messageInput = page.getByRole('textbox', { name: 'Message input' });
|
||||
await messageInput.click();
|
||||
await messageInput.pressSequentially('+');
|
||||
|
||||
const addedModelSearch = page.getByPlaceholder(
|
||||
'Add a model or preset for an additional response',
|
||||
);
|
||||
await expect(addedModelSearch).toBeVisible();
|
||||
await addedModelSearch.fill(agentName);
|
||||
|
||||
const agentOption = page.locator('button[id^="add-convo-item-"]').filter({ hasText: agentName });
|
||||
await expect(agentOption).toHaveCount(1);
|
||||
await agentOption.click();
|
||||
|
||||
await expect(page.getByText(`+ ${agentName}`, { exact: true })).toBeVisible();
|
||||
await expect(messageInput).toHaveValue('');
|
||||
}
|
||||
|
||||
async function selectPrimaryAgent(page: Page, agentName: string) {
|
||||
const form = await openAgentBuilder(page);
|
||||
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
|
||||
await page.getByRole('option', { name: agentName, exact: true }).click();
|
||||
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
}
|
||||
|
||||
async function settleCleanup(tasks: Promise<unknown>[]) {
|
||||
const results = await Promise.allSettled(tasks);
|
||||
const failures = results.filter(
|
||||
(result): result is PromiseRejectedResult => result.status === 'rejected',
|
||||
);
|
||||
if (failures.length > 0) {
|
||||
throw failures[0].reason;
|
||||
}
|
||||
}
|
||||
|
||||
test.describe('added agent skills', () => {
|
||||
test('keeps an added agent deployment-skill catalog scoped to that agent', async ({ page }) => {
|
||||
test.setTimeout(120000);
|
||||
|
||||
const primaryAgentName = uniqueAgentName('E2E Primary Skill Agent');
|
||||
const addedAgentName = uniqueAgentName('E2E Added Skill Agent');
|
||||
let primaryAgentId: string | undefined;
|
||||
let addedAgentId: string | undefined;
|
||||
|
||||
try {
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
const token = await getAccessToken(page);
|
||||
const deploymentSkill = await getDeploymentSkill(page, token);
|
||||
|
||||
const primaryAgent = await requestJson<AgentResponse>(page, {
|
||||
path: '/api/agents',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name: primaryAgentName,
|
||||
description: 'Primary agent without skills for added-agent scope coverage.',
|
||||
instructions: 'Respond as the primary agent.',
|
||||
provider: 'Mock Provider A',
|
||||
model: 'mock-model-a',
|
||||
model_parameters: {},
|
||||
skills_enabled: false,
|
||||
skills: [],
|
||||
},
|
||||
});
|
||||
primaryAgentId = primaryAgent.id;
|
||||
|
||||
const addedAgent = await requestJson<AgentResponse>(page, {
|
||||
path: '/api/agents',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name: addedAgentName,
|
||||
description: 'Persisted added-agent deployment skill regression fixture.',
|
||||
instructions: 'Use the configured deployment skill when responding.',
|
||||
provider: 'Mock Provider B',
|
||||
model: 'mock-model-b',
|
||||
model_parameters: {},
|
||||
skills_enabled: true,
|
||||
skills: [deploymentSkill._id],
|
||||
},
|
||||
});
|
||||
addedAgentId = addedAgent.id;
|
||||
expect(addedAgent).toMatchObject({
|
||||
name: addedAgentName,
|
||||
skills_enabled: true,
|
||||
skills: [deploymentSkill._id],
|
||||
});
|
||||
|
||||
await selectPrimaryAgent(page, primaryAgentName);
|
||||
await selectAddedAgent(page, addedAgentName);
|
||||
|
||||
const scopedAssertion = `${ASSERTION_MARKER}${primaryAgentId}=!${DEPLOYMENT_SKILL_NAME};${addedAgentId}=${DEPLOYMENT_SKILL_NAME}`;
|
||||
const response = await sendMessage(
|
||||
page,
|
||||
[scopedAssertion, 'Verify the persisted added-agent skill catalog.'].join('\n'),
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
const payload = response.request().postDataJSON() as AgentChatPayload;
|
||||
expect(payload.addedConvo).toMatchObject({
|
||||
agent_id: addedAgentId,
|
||||
endpoint: 'agents',
|
||||
});
|
||||
|
||||
const parallelGroup = page
|
||||
.getByTestId('messages-view')
|
||||
.locator('.sibling-content-group')
|
||||
.last();
|
||||
const addedAgentColumn = parallelGroup
|
||||
.locator(':scope > div')
|
||||
.filter({ hasText: addedAgentName });
|
||||
await expect(addedAgentColumn).toHaveCount(1);
|
||||
await expect(addedAgentColumn).toContainText(
|
||||
`${ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}`,
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
|
||||
const primaryColumn = parallelGroup
|
||||
.locator(':scope > div')
|
||||
.filter({ hasText: primaryAgentName });
|
||||
await expect(primaryColumn).toHaveCount(1);
|
||||
await expect(primaryColumn).toContainText(`${ASSERTION_FINAL_TEXT}: none`, {
|
||||
timeout: 30000,
|
||||
});
|
||||
} finally {
|
||||
await settleCleanup([cleanupAgent(page, primaryAgentId), cleanupAgent(page, addedAgentId)]);
|
||||
}
|
||||
});
|
||||
});
|
||||
810
e2e/specs/mock/agent-skills.spec.ts
Normal file
810
e2e/specs/mock/agent-skills.spec.ts
Normal file
|
|
@ -0,0 +1,810 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import type { Locator, Page, Request, Response } from '@playwright/test';
|
||||
import type { AgentDetail } from './agents.helpers';
|
||||
import {
|
||||
cleanupAgent,
|
||||
openAgentBuilder,
|
||||
selectMockModel,
|
||||
uniqueAgentName,
|
||||
waitForPersistedAgent,
|
||||
} from './agents.helpers';
|
||||
import { fetchJson, getAccessToken, requestJson, sendMessage } from './helpers';
|
||||
|
||||
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
|
||||
const SKILL_ASSERTION_MARKER = 'E2E_ASSERT_SKILLS:';
|
||||
const SKILL_ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
|
||||
const MANUAL_SKILL_ASSERTION_MARKER = 'E2E_ASSERT_MANUAL_SKILL:';
|
||||
const MANUAL_SKILL_ASSERTION_FINAL_TEXT = 'E2E manual skill assertion passed';
|
||||
const SKILL_TOOL_INVOCATION_MARKER = 'E2E_INVOKE_SKILL:';
|
||||
const SKILL_TOOL_ASSERTION_FINAL_TEXT = 'E2E skill tool assertion passed';
|
||||
const SKILL_PICKER_PLACEHOLDER = 'Select a Skill by name';
|
||||
const NO_SKILLS_FOUND_TEXT = 'No skills found';
|
||||
const INLINE_SKILL_DESCRIPTION =
|
||||
'Use this inline skill to verify Agent Builder skill selection and persistence.';
|
||||
|
||||
type SkillSummary = {
|
||||
_id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
source?: string;
|
||||
sourceMetadata?: {
|
||||
deployment?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
type SkillAgentDetail = AgentDetail & {
|
||||
skills?: string[];
|
||||
skills_enabled?: boolean;
|
||||
};
|
||||
|
||||
type AgentSkillPayload = {
|
||||
name?: string;
|
||||
skills?: string[];
|
||||
skills_enabled?: boolean;
|
||||
manualSkills?: string[];
|
||||
};
|
||||
|
||||
const uniqueSkillName = (prefix: string) =>
|
||||
`${prefix}-${Date.now().toString(36)}-${Math.floor(Math.random() * 1e4)}`;
|
||||
|
||||
async function createInlineSkill(page: Page, token: string, name: string): Promise<SkillSummary> {
|
||||
return requestJson<SkillSummary>(page, {
|
||||
path: '/api/skills',
|
||||
token,
|
||||
method: 'POST',
|
||||
body: {
|
||||
name,
|
||||
description: INLINE_SKILL_DESCRIPTION,
|
||||
body: `# ${name}\n\nUse this skill in Agent Builder end-to-end tests.`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function createInlineSkills(
|
||||
page: Page,
|
||||
token: string,
|
||||
names: string[],
|
||||
onCreated?: (skill: SkillSummary) => void,
|
||||
): Promise<SkillSummary[]> {
|
||||
const created: SkillSummary[] = [];
|
||||
const batchSize = 10;
|
||||
for (let index = 0; index < names.length; index += batchSize) {
|
||||
const batch = await Promise.allSettled(
|
||||
names.slice(index, index + batchSize).map((name) => createInlineSkill(page, token, name)),
|
||||
);
|
||||
for (const result of batch) {
|
||||
if (result.status === 'fulfilled') {
|
||||
created.push(result.value);
|
||||
onCreated?.(result.value);
|
||||
}
|
||||
}
|
||||
const failed = batch.find((result) => result.status === 'rejected');
|
||||
if (failed?.status === 'rejected') {
|
||||
throw failed.reason;
|
||||
}
|
||||
}
|
||||
return created;
|
||||
}
|
||||
|
||||
async function getDeploymentSkill(page: Page, token: string): Promise<SkillSummary> {
|
||||
const result = await fetchJson<{ skills?: SkillSummary[] }>(
|
||||
page,
|
||||
`/api/skills?search=${encodeURIComponent(DEPLOYMENT_SKILL_NAME)}&limit=10`,
|
||||
token,
|
||||
);
|
||||
const skill = result.skills?.find((item) => item.name === DEPLOYMENT_SKILL_NAME);
|
||||
expect(skill, 'Expected the deployment skill fixture to be accessible').toMatchObject({
|
||||
name: DEPLOYMENT_SKILL_NAME,
|
||||
source: 'deployment',
|
||||
sourceMetadata: { deployment: true },
|
||||
});
|
||||
return skill!;
|
||||
}
|
||||
|
||||
async function deleteSkill(page: Page, token: string, skillId: string): Promise<void> {
|
||||
await requestJson(page, {
|
||||
path: `/api/skills/${encodeURIComponent(skillId)}`,
|
||||
token,
|
||||
method: 'DELETE',
|
||||
});
|
||||
}
|
||||
|
||||
async function deleteSkills(page: Page, token: string, skillIds: string[]): Promise<void> {
|
||||
const batchSize = 10;
|
||||
const failures: unknown[] = [];
|
||||
for (let index = 0; index < skillIds.length; index += batchSize) {
|
||||
const results = await Promise.allSettled(
|
||||
skillIds.slice(index, index + batchSize).map((skillId) => deleteSkill(page, token, skillId)),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status === 'rejected') {
|
||||
failures.push(result.reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
throw failures[0];
|
||||
}
|
||||
}
|
||||
|
||||
async function settleCleanup(tasks: Promise<unknown>[]): Promise<void> {
|
||||
const results = await Promise.allSettled(tasks);
|
||||
const failures = results.filter(
|
||||
(result): result is PromiseRejectedResult => result.status === 'rejected',
|
||||
);
|
||||
if (failures.length > 0) {
|
||||
throw failures[0].reason;
|
||||
}
|
||||
}
|
||||
|
||||
async function openSkillsDialog(page: Page, form: Locator): Promise<Locator> {
|
||||
await form.getByRole('button', { name: 'Add Skills', exact: true }).click();
|
||||
const dialog = page
|
||||
.getByRole('dialog')
|
||||
.filter({ hasText: 'Browse and add skills to your agent.' });
|
||||
await expect(dialog).toBeVisible();
|
||||
return dialog;
|
||||
}
|
||||
|
||||
async function skillCard(dialog: Locator, name: string): Promise<Locator> {
|
||||
const item = dialog
|
||||
.getByRole('list', { name: 'Skills' })
|
||||
.getByRole('listitem')
|
||||
.filter({ hasText: name });
|
||||
await expect(item).toHaveCount(1);
|
||||
const card = item.getByRole('button').filter({ hasText: name }).first();
|
||||
await expect(card).toBeVisible();
|
||||
return card;
|
||||
}
|
||||
|
||||
async function selectSkill(dialog: Locator, name: string): Promise<void> {
|
||||
const card = await skillCard(dialog, name);
|
||||
await card.click();
|
||||
await expect(card).toHaveAttribute('aria-pressed', 'true');
|
||||
}
|
||||
|
||||
async function waitForAgentMutation(
|
||||
page: Page,
|
||||
method: 'POST' | 'PATCH',
|
||||
agentId?: string,
|
||||
): Promise<Response> {
|
||||
return page.waitForResponse(
|
||||
(response) => {
|
||||
const { pathname } = new URL(response.url());
|
||||
const expectedPath = agentId ? `/api/agents/${agentId}` : '/api/agents';
|
||||
return response.request().method() === method && pathname === expectedPath;
|
||||
},
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
}
|
||||
|
||||
async function selectAgent(page: Page, form: Locator, agentName: string): Promise<void> {
|
||||
await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
|
||||
await page.getByRole('option', { name: agentName, exact: true }).click();
|
||||
await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
|
||||
}
|
||||
|
||||
async function openSkillPicker(page: Page): Promise<Locator> {
|
||||
const messageInput = page.getByRole('textbox', { name: 'Message input' });
|
||||
await messageInput.click();
|
||||
await messageInput.fill('');
|
||||
await messageInput.pressSequentially('$');
|
||||
const searchInput = page.getByPlaceholder(SKILL_PICKER_PLACEHOLDER);
|
||||
await expect(searchInput).toBeVisible();
|
||||
return searchInput;
|
||||
}
|
||||
|
||||
function skillPickerOption(page: Page, name: string): Locator {
|
||||
return page.locator('button[id^="skill-item-"]').filter({ hasText: name });
|
||||
}
|
||||
|
||||
async function expectSkillInPicker(page: Page, searchInput: Locator, name: string): Promise<void> {
|
||||
await searchInput.fill(name);
|
||||
await expect(skillPickerOption(page, name)).toHaveCount(1);
|
||||
await expect(skillPickerOption(page, name)).toBeVisible();
|
||||
}
|
||||
|
||||
async function expectSkillAbsentFromPicker(
|
||||
page: Page,
|
||||
searchInput: Locator,
|
||||
name: string,
|
||||
): Promise<void> {
|
||||
await searchInput.fill(name);
|
||||
await expect(
|
||||
searchInput.locator('..').getByText(NO_SKILLS_FOUND_TEXT, { exact: true }),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
await expect(skillPickerOption(page, name)).toHaveCount(0);
|
||||
}
|
||||
|
||||
async function closeSkillPicker(page: Page, searchInput: Locator): Promise<void> {
|
||||
await searchInput.press('Escape');
|
||||
await expect(searchInput).toBeHidden();
|
||||
await page.getByRole('textbox', { name: 'Message input' }).fill('');
|
||||
}
|
||||
|
||||
test.describe('Agent Builder skills', () => {
|
||||
test('creates an agent with deployment and inline skills, then removes and persists one', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(180000);
|
||||
|
||||
const agentName = uniqueAgentName('E2E Agent Skills');
|
||||
const agentDescription = 'Agent Builder skill selection end-to-end coverage.';
|
||||
const inlineSkillName = uniqueSkillName('e2e-agent-inline');
|
||||
const unselectedSkillName = uniqueSkillName('e2e-agent-unselected');
|
||||
let createdAgentId: string | undefined;
|
||||
const createdSkillIds: string[] = [];
|
||||
|
||||
await page.goto('/c/new', { timeout: 10000 });
|
||||
const token = await getAccessToken(page);
|
||||
|
||||
try {
|
||||
const deploymentSkill = await getDeploymentSkill(page, token);
|
||||
const [inlineSkill] = await createInlineSkills(
|
||||
page,
|
||||
token,
|
||||
[inlineSkillName, unselectedSkillName],
|
||||
(skill) => createdSkillIds.push(skill._id),
|
||||
);
|
||||
|
||||
let form = await openAgentBuilder(page);
|
||||
await form.getByLabel('Agent name').fill(agentName);
|
||||
await form.getByLabel('Agent description').fill(agentDescription);
|
||||
await selectMockModel(page, true);
|
||||
form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
const dialog = await openSkillsDialog(page, form);
|
||||
await selectSkill(dialog, DEPLOYMENT_SKILL_NAME);
|
||||
await selectSkill(dialog, inlineSkillName);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(dialog).toBeHidden();
|
||||
|
||||
const createResponsePromise = waitForAgentMutation(page, 'POST');
|
||||
await form.getByRole('button', { name: 'Create', exact: true }).click();
|
||||
const createResponse = await createResponsePromise;
|
||||
expect(createResponse.status(), await createResponse.text()).toBe(201);
|
||||
|
||||
const createdAgent = (await createResponse.json()) as SkillAgentDetail;
|
||||
createdAgentId = createdAgent.id;
|
||||
const createPayload = createResponse.request().postDataJSON() as AgentSkillPayload;
|
||||
expect(createPayload.skills_enabled).toBe(true);
|
||||
expect(createPayload.skills).toHaveLength(2);
|
||||
expect(createPayload.skills).toEqual(
|
||||
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
|
||||
);
|
||||
|
||||
expect(createdAgent).toMatchObject({
|
||||
id: createdAgentId,
|
||||
name: agentName,
|
||||
skills_enabled: true,
|
||||
});
|
||||
expect(createdAgent.skills).toEqual(
|
||||
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
|
||||
);
|
||||
|
||||
const persistedAgent = (await waitForPersistedAgent(
|
||||
page,
|
||||
agentName,
|
||||
agentDescription,
|
||||
)) as SkillAgentDetail;
|
||||
expect(persistedAgent.skills_enabled).toBe(true);
|
||||
expect(persistedAgent.skills).toEqual(
|
||||
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
|
||||
);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeVisible();
|
||||
await expect(form.getByText(inlineSkillName, { exact: true })).toBeVisible();
|
||||
|
||||
const retainedDescription = `${agentDescription} Saved with both skills still selected.`;
|
||||
await form.getByLabel('Agent description').fill(retainedDescription);
|
||||
const retentionResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
|
||||
await form.getByRole('button', { name: 'Save', exact: true }).click();
|
||||
const retentionResponse = await retentionResponsePromise;
|
||||
expect(retentionResponse.status(), await retentionResponse.text()).toBe(200);
|
||||
|
||||
const retainedAgent = (await retentionResponse.json()) as SkillAgentDetail;
|
||||
const retentionPayload = retentionResponse.request().postDataJSON() as AgentSkillPayload;
|
||||
expect(retentionPayload.skills_enabled).toBe(true);
|
||||
expect(retentionPayload.skills).toHaveLength(2);
|
||||
expect(retentionPayload.skills).toEqual(
|
||||
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
|
||||
);
|
||||
expect(retainedAgent.skills_enabled).toBe(true);
|
||||
expect(retainedAgent.skills).toHaveLength(2);
|
||||
expect(retainedAgent.skills).toEqual(
|
||||
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
|
||||
);
|
||||
|
||||
const expandedRetainedAgent = await fetchJson<SkillAgentDetail>(
|
||||
page,
|
||||
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
|
||||
token,
|
||||
);
|
||||
expect(expandedRetainedAgent.skills_enabled).toBe(true);
|
||||
expect(expandedRetainedAgent.skills).toHaveLength(2);
|
||||
expect(expandedRetainedAgent.skills).toEqual(
|
||||
expect.arrayContaining([deploymentSkill._id, inlineSkill._id]),
|
||||
);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeVisible();
|
||||
await expect(form.getByText(inlineSkillName, { exact: true })).toBeVisible();
|
||||
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
const pickerSearch = await openSkillPicker(page);
|
||||
await expectSkillInPicker(page, pickerSearch, DEPLOYMENT_SKILL_NAME);
|
||||
await expectSkillInPicker(page, pickerSearch, inlineSkillName);
|
||||
await expectSkillAbsentFromPicker(page, pickerSearch, unselectedSkillName);
|
||||
await closeSkillPicker(page, pickerSearch);
|
||||
|
||||
const runtimeResponse = await sendMessage(
|
||||
page,
|
||||
[
|
||||
`${SKILL_ASSERTION_MARKER}*${DEPLOYMENT_SKILL_NAME},${inlineSkillName},!${unselectedSkillName}`,
|
||||
'Verify the persisted Agent Builder skill allowlist at runtime.',
|
||||
].join('\n'),
|
||||
);
|
||||
expect(runtimeResponse.ok()).toBeTruthy();
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`${SKILL_ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}, ${inlineSkillName}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
|
||||
const inlineToolResponse = await sendMessage(
|
||||
page,
|
||||
`${SKILL_TOOL_INVOCATION_MARKER}${inlineSkillName}`,
|
||||
);
|
||||
expect(inlineToolResponse.ok()).toBeTruthy();
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`${SKILL_TOOL_ASSERTION_FINAL_TEXT}: ${inlineSkillName}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
|
||||
const deploymentToolResponse = await sendMessage(
|
||||
page,
|
||||
`${SKILL_TOOL_INVOCATION_MARKER}${DEPLOYMENT_SKILL_NAME}`,
|
||||
);
|
||||
expect(deploymentToolResponse.ok()).toBeTruthy();
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`${SKILL_TOOL_ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
|
||||
const manualPickerSearch = await openSkillPicker(page);
|
||||
await expectSkillInPicker(page, manualPickerSearch, inlineSkillName);
|
||||
await skillPickerOption(page, inlineSkillName).click();
|
||||
await expect(manualPickerSearch).toBeHidden();
|
||||
await expect(
|
||||
page
|
||||
.getByRole('list', { name: 'Skills queued for next submission' })
|
||||
.getByText(inlineSkillName, { exact: true }),
|
||||
).toBeVisible();
|
||||
|
||||
const manualResponse = await sendMessage(
|
||||
page,
|
||||
`${MANUAL_SKILL_ASSERTION_MARKER}${inlineSkillName}`,
|
||||
);
|
||||
expect(manualResponse.ok()).toBeTruthy();
|
||||
expect((manualResponse.request().postDataJSON() as AgentSkillPayload).manualSkills).toEqual([
|
||||
inlineSkillName,
|
||||
]);
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`${MANUAL_SKILL_ASSERTION_FINAL_TEXT}: ${inlineSkillName}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
const deploymentRow = form.locator('li').filter({ hasText: DEPLOYMENT_SKILL_NAME });
|
||||
await expect(deploymentRow).toHaveCount(1);
|
||||
await deploymentRow.hover();
|
||||
await deploymentRow.getByRole('button', { name: 'Remove from agent' }).click();
|
||||
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeHidden();
|
||||
|
||||
const updateResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
|
||||
await form.getByRole('button', { name: 'Save', exact: true }).click();
|
||||
const updateResponse = await updateResponsePromise;
|
||||
expect(updateResponse.status(), await updateResponse.text()).toBe(200);
|
||||
|
||||
const updatePayload = updateResponse.request().postDataJSON() as AgentSkillPayload;
|
||||
expect(updatePayload).toMatchObject({
|
||||
skills: [inlineSkill._id],
|
||||
skills_enabled: true,
|
||||
});
|
||||
|
||||
const updatedAgent = await fetchJson<SkillAgentDetail>(
|
||||
page,
|
||||
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
|
||||
token,
|
||||
);
|
||||
expect(updatedAgent.skills).toEqual([inlineSkill._id]);
|
||||
expect(updatedAgent.skills_enabled).toBe(true);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByText(inlineSkillName, { exact: true })).toBeVisible();
|
||||
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeHidden();
|
||||
|
||||
const finalSkillRow = form.locator('li').filter({ hasText: inlineSkillName });
|
||||
await expect(finalSkillRow).toHaveCount(1);
|
||||
await finalSkillRow.hover();
|
||||
await finalSkillRow.getByRole('button', { name: 'Remove from agent' }).click();
|
||||
await expect(form.getByText(inlineSkillName, { exact: true })).toBeHidden();
|
||||
|
||||
const disableResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
|
||||
await form.getByRole('button', { name: 'Save', exact: true }).click();
|
||||
const disableResponse = await disableResponsePromise;
|
||||
expect(disableResponse.status(), await disableResponse.text()).toBe(200);
|
||||
expect(disableResponse.request().postDataJSON()).toMatchObject({
|
||||
skills: [],
|
||||
skills_enabled: false,
|
||||
});
|
||||
|
||||
const disabledAgent = await fetchJson<SkillAgentDetail>(
|
||||
page,
|
||||
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
|
||||
token,
|
||||
);
|
||||
expect(disabledAgent.skills).toEqual([]);
|
||||
expect(disabledAgent.skills_enabled).toBe(false);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByText(DEPLOYMENT_SKILL_NAME, { exact: true })).toBeHidden();
|
||||
await expect(form.getByText(inlineSkillName, { exact: true })).toBeHidden();
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
|
||||
const disabledPickerSearch = await openSkillPicker(page);
|
||||
await expectSkillAbsentFromPicker(page, disabledPickerSearch, DEPLOYMENT_SKILL_NAME);
|
||||
await expectSkillAbsentFromPicker(page, disabledPickerSearch, inlineSkillName);
|
||||
await closeSkillPicker(page, disabledPickerSearch);
|
||||
|
||||
const disabledRuntimeResponse = await sendMessage(
|
||||
page,
|
||||
`${SKILL_ASSERTION_MARKER}!${DEPLOYMENT_SKILL_NAME},!${inlineSkillName}`,
|
||||
);
|
||||
expect(disabledRuntimeResponse.ok()).toBeTruthy();
|
||||
await expect(
|
||||
page.getByTestId('messages-view').getByText(`${SKILL_ASSERTION_FINAL_TEXT}: none`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
} finally {
|
||||
await settleCleanup([
|
||||
cleanupAgent(page, createdAgentId),
|
||||
deleteSkills(page, token, createdSkillIds),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
test('persists explicit use-all semantics and includes skills created later', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(180000);
|
||||
|
||||
const agentName = uniqueAgentName('E2E Agent All Skills');
|
||||
const agentDescription = 'Agent Builder explicit use-all-skills end-to-end coverage.';
|
||||
const futureSkillName = uniqueSkillName('e2e-agent-future');
|
||||
let createdAgentId: string | undefined;
|
||||
let futureSkill: SkillSummary | undefined;
|
||||
|
||||
await page.goto('/c/new', { timeout: 10000 });
|
||||
const token = await getAccessToken(page);
|
||||
|
||||
try {
|
||||
let form = await openAgentBuilder(page);
|
||||
await form.getByLabel('Agent name').fill(agentName);
|
||||
await form.getByLabel('Agent description').fill(agentDescription);
|
||||
await selectMockModel(page, true);
|
||||
form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
const useAllSwitch = form.getByRole('switch', { name: 'Use all skills' });
|
||||
await expect(useAllSwitch).toHaveAttribute('aria-checked', 'false');
|
||||
await useAllSwitch.click();
|
||||
await expect(useAllSwitch).toHaveAttribute('aria-checked', 'true');
|
||||
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toHaveCount(0);
|
||||
|
||||
const createResponsePromise = waitForAgentMutation(page, 'POST');
|
||||
await form.getByRole('button', { name: 'Create', exact: true }).click();
|
||||
const createResponse = await createResponsePromise;
|
||||
expect(createResponse.status(), await createResponse.text()).toBe(201);
|
||||
|
||||
const createdAgent = (await createResponse.json()) as SkillAgentDetail;
|
||||
createdAgentId = createdAgent.id;
|
||||
const createPayload = createResponse.request().postDataJSON() as AgentSkillPayload;
|
||||
expect(createPayload).toMatchObject({
|
||||
skills: [],
|
||||
skills_enabled: true,
|
||||
});
|
||||
|
||||
expect(createdAgent).toMatchObject({
|
||||
id: createdAgentId,
|
||||
skills: [],
|
||||
skills_enabled: true,
|
||||
});
|
||||
|
||||
const persistedAgent = (await waitForPersistedAgent(
|
||||
page,
|
||||
agentName,
|
||||
agentDescription,
|
||||
)) as SkillAgentDetail;
|
||||
expect(persistedAgent.skills).toEqual([]);
|
||||
expect(persistedAgent.skills_enabled).toBe(true);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByRole('switch', { name: 'Use all skills' })).toHaveAttribute(
|
||||
'aria-checked',
|
||||
'true',
|
||||
);
|
||||
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toHaveCount(0);
|
||||
|
||||
futureSkill = await createInlineSkill(page, token, futureSkillName);
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
const useAllPickerSearch = await openSkillPicker(page);
|
||||
await expectSkillInPicker(page, useAllPickerSearch, futureSkillName);
|
||||
await closeSkillPicker(page, useAllPickerSearch);
|
||||
|
||||
const runtimeResponse = await sendMessage(
|
||||
page,
|
||||
[
|
||||
`${SKILL_ASSERTION_MARKER}${DEPLOYMENT_SKILL_NAME},${futureSkillName}`,
|
||||
'Verify use-all includes a skill created after the agent was saved.',
|
||||
].join('\n'),
|
||||
);
|
||||
expect(runtimeResponse.ok()).toBeTruthy();
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`${SKILL_ASSERTION_FINAL_TEXT}: ${DEPLOYMENT_SKILL_NAME}, ${futureSkillName}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
const persistedUseAllSwitch = form.getByRole('switch', { name: 'Use all skills' });
|
||||
await persistedUseAllSwitch.click();
|
||||
await expect(persistedUseAllSwitch).toHaveAttribute('aria-checked', 'false');
|
||||
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toBeVisible();
|
||||
|
||||
const updateResponsePromise = waitForAgentMutation(page, 'PATCH', createdAgentId);
|
||||
await form.getByRole('button', { name: 'Save', exact: true }).click();
|
||||
const updateResponse = await updateResponsePromise;
|
||||
expect(updateResponse.status(), await updateResponse.text()).toBe(200);
|
||||
expect(updateResponse.request().postDataJSON()).toMatchObject({
|
||||
skills: [],
|
||||
skills_enabled: false,
|
||||
});
|
||||
|
||||
const disabledAgent = await fetchJson<SkillAgentDetail>(
|
||||
page,
|
||||
`/api/agents/${encodeURIComponent(createdAgentId)}/expanded`,
|
||||
token,
|
||||
);
|
||||
expect(disabledAgent.skills).toEqual([]);
|
||||
expect(disabledAgent.skills_enabled).toBe(false);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByRole('switch', { name: 'Use all skills' })).toHaveAttribute(
|
||||
'aria-checked',
|
||||
'false',
|
||||
);
|
||||
await expect(form.getByRole('button', { name: 'Add Skills', exact: true })).toBeVisible();
|
||||
|
||||
await form.getByRole('button', { name: 'Select Agent' }).click();
|
||||
const disabledPickerSearch = await openSkillPicker(page);
|
||||
await expectSkillAbsentFromPicker(page, disabledPickerSearch, futureSkillName);
|
||||
await closeSkillPicker(page, disabledPickerSearch);
|
||||
|
||||
const disabledRuntimeResponse = await sendMessage(
|
||||
page,
|
||||
[
|
||||
`${SKILL_ASSERTION_MARKER}!${DEPLOYMENT_SKILL_NAME},!${futureSkillName}`,
|
||||
'Verify the master skill toggle hides every skill at runtime.',
|
||||
].join('\n'),
|
||||
);
|
||||
expect(disabledRuntimeResponse.ok()).toBeTruthy();
|
||||
await expect(
|
||||
page.getByTestId('messages-view').getByText(`${SKILL_ASSERTION_FINAL_TEXT}: none`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
} finally {
|
||||
await settleCleanup([
|
||||
cleanupAgent(page, createdAgentId),
|
||||
...(futureSkill ? [deleteSkill(page, token, futureSkill._id)] : []),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
test('creates a skill inside the builder and auto-selects it for the new agent', async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(180000);
|
||||
|
||||
const agentName = uniqueAgentName('E2E Agent Created Skill');
|
||||
const agentDescription = 'Agent Builder inline skill creation end-to-end coverage.';
|
||||
const skillName = uniqueSkillName('e2e-builder-created');
|
||||
let createdAgentId: string | undefined;
|
||||
let createdSkill: SkillSummary | undefined;
|
||||
let agentPostCount = 0;
|
||||
const countAgentPosts = (request: Request) => {
|
||||
if (request.method() === 'POST' && new URL(request.url()).pathname === '/api/agents') {
|
||||
agentPostCount += 1;
|
||||
}
|
||||
};
|
||||
page.on('request', countAgentPosts);
|
||||
|
||||
await page.goto('/c/new', { timeout: 10000 });
|
||||
const token = await getAccessToken(page);
|
||||
|
||||
try {
|
||||
let form = await openAgentBuilder(page);
|
||||
await form.getByLabel('Agent name').fill(agentName);
|
||||
await form.getByLabel('Agent description').fill(agentDescription);
|
||||
await selectMockModel(page, true);
|
||||
form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
const skillsDialog = await openSkillsDialog(page, form);
|
||||
await skillsDialog.getByRole('button', { name: 'Create Skill', exact: true }).click();
|
||||
|
||||
const createSkillDialog = page
|
||||
.getByRole('dialog')
|
||||
.filter({ hasText: 'Write skill instructions' });
|
||||
await expect(createSkillDialog).toBeVisible();
|
||||
await createSkillDialog.getByLabel('Name').fill(skillName);
|
||||
await createSkillDialog.getByLabel('Description').fill(INLINE_SKILL_DESCRIPTION);
|
||||
await createSkillDialog
|
||||
.getByLabel('Instructions')
|
||||
.fill(`# ${skillName}\n\nCreated inline from Agent Builder.`);
|
||||
|
||||
const skillResponsePromise = page.waitForResponse(
|
||||
(response) =>
|
||||
response.request().method() === 'POST' &&
|
||||
new URL(response.url()).pathname === '/api/skills',
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
await createSkillDialog.getByRole('button', { name: 'Create', exact: true }).click();
|
||||
const skillResponse = await skillResponsePromise;
|
||||
expect(skillResponse.status(), await skillResponse.text()).toBe(201);
|
||||
createdSkill = (await skillResponse.json()) as SkillSummary;
|
||||
expect(skillResponse.request().postDataJSON()).toMatchObject({
|
||||
name: skillName,
|
||||
description: INLINE_SKILL_DESCRIPTION,
|
||||
});
|
||||
expect(agentPostCount).toBe(0);
|
||||
|
||||
await expect(createSkillDialog).toBeHidden();
|
||||
const createdCard = await skillCard(skillsDialog, skillName);
|
||||
await expect(createdCard).toHaveAttribute('aria-pressed', 'true');
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(skillsDialog).toBeHidden();
|
||||
await expect(form.getByText(skillName, { exact: true })).toBeVisible();
|
||||
|
||||
const createAgentResponsePromise = waitForAgentMutation(page, 'POST');
|
||||
await form.getByRole('button', { name: 'Create', exact: true }).click();
|
||||
const createAgentResponse = await createAgentResponsePromise;
|
||||
expect(createAgentResponse.status(), await createAgentResponse.text()).toBe(201);
|
||||
expect(agentPostCount).toBe(1);
|
||||
const createdAgent = (await createAgentResponse.json()) as SkillAgentDetail;
|
||||
createdAgentId = createdAgent.id;
|
||||
expect(createAgentResponse.request().postDataJSON()).toMatchObject({
|
||||
skills: [createdSkill._id],
|
||||
skills_enabled: true,
|
||||
});
|
||||
|
||||
expect(createdAgent).toMatchObject({
|
||||
id: createdAgentId,
|
||||
skills: [createdSkill._id],
|
||||
skills_enabled: true,
|
||||
});
|
||||
|
||||
const persistedAgent = (await waitForPersistedAgent(
|
||||
page,
|
||||
agentName,
|
||||
agentDescription,
|
||||
)) as SkillAgentDetail;
|
||||
expect(persistedAgent.skills).toEqual([createdSkill._id]);
|
||||
expect(persistedAgent.skills_enabled).toBe(true);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByText(skillName, { exact: true })).toBeVisible();
|
||||
} finally {
|
||||
page.off('request', countAgentPosts);
|
||||
await settleCleanup([
|
||||
cleanupAgent(page, createdAgentId),
|
||||
...(createdSkill ? [deleteSkill(page, token, createdSkill._id)] : []),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
test('discovers and selects a skill beyond the first 100 accessible skills', async ({ page }) => {
|
||||
test.setTimeout(240000);
|
||||
|
||||
const agentName = uniqueAgentName('E2E Agent Paginated Skill');
|
||||
const agentDescription = 'Agent Builder paginated skill discovery end-to-end coverage.';
|
||||
const targetSkillName = uniqueSkillName('e2e-agent-off-page');
|
||||
const fillerPrefix = uniqueSkillName('e2e-agent-page');
|
||||
let createdAgentId: string | undefined;
|
||||
const createdSkillIds: string[] = [];
|
||||
|
||||
await page.goto('/c/new', { timeout: 10000 });
|
||||
const token = await getAccessToken(page);
|
||||
|
||||
try {
|
||||
const targetSkill = await createInlineSkill(page, token, targetSkillName);
|
||||
createdSkillIds.push(targetSkill._id);
|
||||
const fillerNames = Array.from(
|
||||
{ length: 100 },
|
||||
(_, index) => `${fillerPrefix}-${String(index).padStart(3, '0')}`,
|
||||
);
|
||||
await createInlineSkills(page, token, fillerNames, (skill) =>
|
||||
createdSkillIds.push(skill._id),
|
||||
);
|
||||
|
||||
const firstPage = await fetchJson<{
|
||||
skills: SkillSummary[];
|
||||
has_more: boolean;
|
||||
after: string | null;
|
||||
}>(page, '/api/skills?limit=100', token);
|
||||
expect(firstPage.has_more).toBe(true);
|
||||
expect(firstPage.after).toBeTruthy();
|
||||
expect(firstPage.skills.some((skill) => skill._id === targetSkill._id)).toBe(false);
|
||||
|
||||
let form = await openAgentBuilder(page);
|
||||
await form.getByLabel('Agent name').fill(agentName);
|
||||
await form.getByLabel('Agent description').fill(agentDescription);
|
||||
await selectMockModel(page, true);
|
||||
form = page.getByRole('form', { name: 'Agent configuration form' });
|
||||
|
||||
const nextPageResponsePromise = page.waitForResponse(
|
||||
(response) => {
|
||||
const url = new URL(response.url());
|
||||
return url.pathname === '/api/skills' && url.searchParams.has('cursor');
|
||||
},
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
const dialog = await openSkillsDialog(page, form);
|
||||
const nextPageResponse = await nextPageResponsePromise;
|
||||
expect(nextPageResponse.ok(), await nextPageResponse.text()).toBe(true);
|
||||
|
||||
await dialog.getByRole('textbox', { name: 'Search skills...' }).fill(targetSkillName);
|
||||
await selectSkill(dialog, targetSkillName);
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(dialog).toBeHidden();
|
||||
await expect(form.getByText(targetSkillName, { exact: true })).toBeVisible();
|
||||
|
||||
const createResponsePromise = waitForAgentMutation(page, 'POST');
|
||||
await form.getByRole('button', { name: 'Create', exact: true }).click();
|
||||
const createResponse = await createResponsePromise;
|
||||
expect(createResponse.status(), await createResponse.text()).toBe(201);
|
||||
const createdAgent = (await createResponse.json()) as SkillAgentDetail;
|
||||
createdAgentId = createdAgent.id;
|
||||
expect(createResponse.request().postDataJSON()).toMatchObject({
|
||||
skills: [targetSkill._id],
|
||||
skills_enabled: true,
|
||||
});
|
||||
|
||||
const persistedAgent = (await waitForPersistedAgent(
|
||||
page,
|
||||
agentName,
|
||||
agentDescription,
|
||||
)) as SkillAgentDetail;
|
||||
expect(persistedAgent.skills).toEqual([targetSkill._id]);
|
||||
expect(persistedAgent.skills_enabled).toBe(true);
|
||||
|
||||
form = await openAgentBuilder(page);
|
||||
await selectAgent(page, form, agentName);
|
||||
await expect(form.getByText(targetSkillName, { exact: true })).toBeVisible();
|
||||
} finally {
|
||||
await settleCleanup([
|
||||
cleanupAgent(page, createdAgentId),
|
||||
deleteSkills(page, token, createdSkillIds),
|
||||
]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -12,9 +12,11 @@ import {
|
|||
} from './helpers';
|
||||
|
||||
const MODEL_SPEC_LABEL = 'E2E Skill Scope';
|
||||
const ASSERTION_MARKER = 'E2E_ASSERT_MODEL_SPEC_SKILLS';
|
||||
const ASSERTION_FINAL_TEXT = 'E2E model spec skill assertion passed';
|
||||
const ASSERTION_MARKER = 'E2E_ASSERT_SKILLS:';
|
||||
const ASSERTION_FINAL_TEXT = 'E2E skill assertion passed';
|
||||
const ACCESSIBLE_SKILL_NAME = 'e2e-model-spec-allowed';
|
||||
const DEPLOYMENT_SKILL_NAME = 'e2e-deployment-skill';
|
||||
const MISSING_SKILL_NAME = 'e2e-model-spec-missing';
|
||||
const INACCESSIBLE_SKILL_NAME = 'e2e-model-spec-inaccessible';
|
||||
const ALWAYS_APPLY_BODY_MARKER = 'E2E_ALWAYS_APPLY_BODY_MARKER';
|
||||
const INACCESSIBLE_AUTHOR_ID = new ObjectId('64f000000000000000000001');
|
||||
|
|
@ -93,7 +95,15 @@ async function seedAccessibleSkill(page: Page, token: string): Promise<SkillDeta
|
|||
});
|
||||
}
|
||||
|
||||
async function seedInaccessibleSkill() {
|
||||
async function deleteAccessibleSkill(page: Page, token: string, skillId: string) {
|
||||
await requestJson<{ deleted: boolean }>(page, {
|
||||
path: `/api/skills/${encodeURIComponent(skillId)}`,
|
||||
token,
|
||||
method: 'DELETE',
|
||||
});
|
||||
}
|
||||
|
||||
async function seedInaccessibleSkill(): Promise<string> {
|
||||
applyRuntimeEnv();
|
||||
if (!process.env.MONGO_URI) {
|
||||
throw new Error('MONGO_URI must be available for model-spec skill mock e2e tests');
|
||||
|
|
@ -141,7 +151,30 @@ async function seedInaccessibleSkill() {
|
|||
});
|
||||
if (skill?._id) {
|
||||
await aclEntries.deleteMany({ resourceType: 'skill', resourceId: skill._id });
|
||||
return skill._id.toString();
|
||||
}
|
||||
throw new Error(`Failed to seed inaccessible skill "${INACCESSIBLE_SKILL_NAME}"`);
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteInaccessibleSkill(skillId: string) {
|
||||
applyRuntimeEnv();
|
||||
if (!process.env.MONGO_URI) {
|
||||
throw new Error('MONGO_URI must be available for model-spec skill mock e2e tests');
|
||||
}
|
||||
|
||||
const client = new MongoClient(process.env.MONGO_URI);
|
||||
await client.connect();
|
||||
try {
|
||||
const db = client.db();
|
||||
const objectId = new ObjectId(skillId);
|
||||
await db.collection('aclentries').deleteMany({
|
||||
resourceType: 'skill',
|
||||
resourceId: objectId,
|
||||
});
|
||||
await db.collection('skills').deleteOne({ _id: objectId });
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
|
|
@ -155,21 +188,60 @@ test.describe('model spec skills', () => {
|
|||
|
||||
await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
|
||||
const token = await getAccessToken(page);
|
||||
const skill = await seedAccessibleSkill(page, token);
|
||||
expect(skill.alwaysApply).toBe(true);
|
||||
await seedInaccessibleSkill();
|
||||
let accessibleSkillId: string | undefined;
|
||||
let inaccessibleSkillId: string | undefined;
|
||||
let testFailure: unknown;
|
||||
|
||||
await selectModelSpec(page, MODEL_SPEC_LABEL);
|
||||
const response = await sendMessage(
|
||||
page,
|
||||
`${ASSERTION_MARKER}\nVerify model-spec skill scope and always-apply frontmatter.`,
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
try {
|
||||
const skill = await seedAccessibleSkill(page, token);
|
||||
accessibleSkillId = skill._id;
|
||||
expect(skill.alwaysApply).toBe(true);
|
||||
inaccessibleSkillId = await seedInaccessibleSkill();
|
||||
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`${ASSERTION_FINAL_TEXT}: ${ACCESSIBLE_SKILL_NAME}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
await selectModelSpec(page, MODEL_SPEC_LABEL);
|
||||
const response = await sendMessage(
|
||||
page,
|
||||
[
|
||||
`${ASSERTION_MARKER}*${ACCESSIBLE_SKILL_NAME},*${DEPLOYMENT_SKILL_NAME},!${MISSING_SKILL_NAME},!${INACCESSIBLE_SKILL_NAME}`,
|
||||
'Verify model-spec skill scope and always-apply frontmatter.',
|
||||
].join('\n'),
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await expect(
|
||||
page
|
||||
.getByTestId('messages-view')
|
||||
.getByText(`${ASSERTION_FINAL_TEXT}: ${ACCESSIBLE_SKILL_NAME}, ${DEPLOYMENT_SKILL_NAME}`),
|
||||
).toBeVisible({ timeout: 30000 });
|
||||
} catch (error) {
|
||||
testFailure = error;
|
||||
} finally {
|
||||
const cleanupTasks: Promise<unknown>[] = [];
|
||||
if (accessibleSkillId) {
|
||||
cleanupTasks.push(deleteAccessibleSkill(page, token, accessibleSkillId));
|
||||
}
|
||||
if (inaccessibleSkillId) {
|
||||
cleanupTasks.push(deleteInaccessibleSkill(inaccessibleSkillId));
|
||||
}
|
||||
const cleanupResults = await Promise.allSettled(cleanupTasks);
|
||||
const cleanupFailures = cleanupResults
|
||||
.filter((result): result is PromiseRejectedResult => result.status === 'rejected')
|
||||
.map((result) => result.reason);
|
||||
if (testFailure && cleanupFailures.length > 0) {
|
||||
testFailure = new AggregateError(
|
||||
[testFailure, ...cleanupFailures],
|
||||
'The model-spec skill assertion and fixture cleanup both failed',
|
||||
);
|
||||
} else if (cleanupFailures.length > 0) {
|
||||
testFailure = new AggregateError(
|
||||
cleanupFailures,
|
||||
'Model-spec skill fixture cleanup failed',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (testFailure) {
|
||||
throw testFailure;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue