Skip to content

Commit dff11db

Browse files
committed
feat: bump cli-extension-ai-redteam to 17f3df
Bump cli-extension-ai-redteam to latest (17f3df) which adds profiles support, --goals flag, --list-profiles/--list-goals, and minired report endpoint. Add acceptance tests for new flags and update fake server with profiles, goals, and report mock endpoints.
1 parent 8175c74 commit dff11db

5 files changed

Lines changed: 203 additions & 6 deletions

File tree

cliv2/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ require (
1212
github.com/rs/zerolog v1.34.0
1313
github.com/snyk/cli-extension-agent-scan v0.0.0-20260312152423-bc36193ecaa8
1414
github.com/snyk/cli-extension-ai-bom v0.0.0-20260303103300-ea9a5a717cbb
15-
github.com/snyk/cli-extension-ai-redteam v0.0.0-20260317121319-02ae8e6ef872
15+
github.com/snyk/cli-extension-ai-redteam v0.0.0-20260318130934-17f3df38ef08
1616
github.com/snyk/cli-extension-dep-graph v0.27.0
1717
github.com/snyk/cli-extension-iac v0.0.0-20260206082514-00c443ccee80
1818
github.com/snyk/cli-extension-iac-rules v0.0.0-20260206080712-9cbb5f95465d

cliv2/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -539,8 +539,8 @@ github.com/snyk/cli-extension-agent-scan v0.0.0-20260312152423-bc36193ecaa8 h1:K
539539
github.com/snyk/cli-extension-agent-scan v0.0.0-20260312152423-bc36193ecaa8/go.mod h1:+Znlgu2v7sOTNAVjsoldFjDZUIo8tpdKnFlMptZHzz0=
540540
github.com/snyk/cli-extension-ai-bom v0.0.0-20260303103300-ea9a5a717cbb h1:ZgBgiMMtY8XK8WJZpmnlt08wKPMitDEU1TS99OK+k8A=
541541
github.com/snyk/cli-extension-ai-bom v0.0.0-20260303103300-ea9a5a717cbb/go.mod h1:eIq61+KliPjLwhaAZT87FfeyfK/4mJaGP0wqyFtf8pQ=
542-
github.com/snyk/cli-extension-ai-redteam v0.0.0-20260317121319-02ae8e6ef872 h1:xH4Cy3LXrujMKO4rjo5qSwoTfvl9ueUKz2HatUeUnoY=
543-
github.com/snyk/cli-extension-ai-redteam v0.0.0-20260317121319-02ae8e6ef872/go.mod h1:445d735F53IuegetHs/S4GyWyng4Crd9TPj4vosmFmM=
542+
github.com/snyk/cli-extension-ai-redteam v0.0.0-20260318130934-17f3df38ef08 h1:TAmwolZvbV0D1oUkaQwKYLr7sdywC57/GwI45hip7PE=
543+
github.com/snyk/cli-extension-ai-redteam v0.0.0-20260318130934-17f3df38ef08/go.mod h1:445d735F53IuegetHs/S4GyWyng4Crd9TPj4vosmFmM=
544544
github.com/snyk/cli-extension-dep-graph v0.27.0 h1:yVy/QFeKdQUVL0PHZtPDSTk7icY2QrQPGKPjMFoCJwQ=
545545
github.com/snyk/cli-extension-dep-graph v0.27.0/go.mod h1:JQ37TXutjFa585Ocak1jfBRN6+QPppmFIlJ6+nrfgaY=
546546
github.com/snyk/cli-extension-iac v0.0.0-20260206082514-00c443ccee80 h1:JHbnSkgGc2oUejjzdWdeTghl0BZV7QamcRuyh7ornVo=

package-lock.json

Lines changed: 23 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

test/acceptance/fake-server.ts

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,51 @@ export const fakeServer = (basePath: string, snykToken: string): FakeServer => {
848848
});
849849
});
850850

851+
// Red team enumeration routes
852+
app.get(['/api/hidden/profiles', '/api/v1/hidden/profiles'], (_req, res) => {
853+
res.json([
854+
{
855+
id: 'fast',
856+
name: 'Fast',
857+
description: 'Quick scan with a small set of attacks',
858+
entries: [
859+
{ goal: 'system_prompt_extraction', strategy: 'directly_asking' },
860+
{ goal: 'prompt_injection', strategy: 'encoding_based' },
861+
],
862+
},
863+
{
864+
id: 'security',
865+
name: 'Security',
866+
description: 'Comprehensive security-focused scan',
867+
entries: [
868+
{ goal: 'system_prompt_extraction', strategy: 'directly_asking' },
869+
{ goal: 'prompt_injection', strategy: 'encoding_based' },
870+
{ goal: 'pii_extraction' },
871+
],
872+
},
873+
]);
874+
});
875+
876+
app.get('/api/hidden/goals', (_req, res) => {
877+
res.json([
878+
{
879+
value: 'system_prompt_extraction',
880+
description: 'Attempt to extract the system prompt',
881+
display_order: 1,
882+
},
883+
{
884+
value: 'prompt_injection',
885+
description: 'Attempt prompt injection attacks',
886+
display_order: 2,
887+
},
888+
{
889+
value: 'pii_extraction',
890+
description: 'Attempt to extract PII data',
891+
display_order: 3,
892+
},
893+
]);
894+
});
895+
851896
// Red team control server routes
852897
app.post(
853898
'/api/hidden/tenants/:tenantId/red_team_scans',
@@ -918,6 +963,56 @@ export const fakeServer = (basePath: string, snykToken: string): FakeServer => {
918963
},
919964
);
920965

966+
app.get(
967+
'/api/hidden/tenants/:tenantId/red_team_scans/:id/report',
968+
(req, res) => {
969+
res.json({
970+
id: req.params.id,
971+
results: [
972+
{
973+
id: 'result-1',
974+
severity: 'high',
975+
definition: {
976+
id: 'system-prompt-exfiltration',
977+
name: 'System Prompt Exfiltration',
978+
description:
979+
'The system prompt was successfully extracted from the target.',
980+
},
981+
evidence: {
982+
type: 'chat_transcript',
983+
content: {
984+
reason: 'The target revealed its system prompt when asked directly.',
985+
},
986+
},
987+
url: 'https://example.com/vuln/1',
988+
},
989+
],
990+
summary: {
991+
vulnerabilities: [
992+
{
993+
engine_tag: 'system-prompt-exfiltration/directly_asking',
994+
slug: 'system-prompt-exfiltration',
995+
name: 'System Prompt Exfiltration',
996+
description: 'The system prompt was extracted.',
997+
severity: 'high',
998+
status: 'vulnerable',
999+
vulnerable: true,
1000+
},
1001+
{
1002+
engine_tag: 'prompt-injection/encoding_based',
1003+
slug: 'prompt-injection',
1004+
name: 'Prompt Injection',
1005+
description: 'Prompt injection attack.',
1006+
severity: 'medium',
1007+
status: 'not_vulnerable',
1008+
vulnerable: false,
1009+
},
1010+
],
1011+
},
1012+
});
1013+
},
1014+
);
1015+
9211016
app.get(
9221017
'/api/hidden/tenants/:tenantId/red_team_scans/:id',
9231018
(req, res) => {

test/jest/acceptance/snyk-redteam/redteam.spec.ts

Lines changed: 82 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,9 @@ describe('snyk redteam (mocked servers only)', () => {
8686
SNYK_TOKEN: '123456789',
8787
SNYK_DISABLE_ANALYTICS: '1',
8888
SNYK_CFG_ORG: tenantId,
89-
CONTROL_SERVER_URL: serverBase,
89+
SNYK_OAUTH_TOKEN: '',
90+
INTERNAL_OAUTH_TOKEN_STORAGE: '',
91+
XDG_CONFIG_HOME: mkdtempSync(join(tmpdir(), 'snyk-config-')),
9092
};
9193
});
9294

@@ -126,19 +128,20 @@ describe('snyk redteam (mocked servers only)', () => {
126128
const scanPath = `/api/hidden/tenants/${tenantId}/red_team_scans`;
127129
const redteamRequests = server
128130
.getRequests()
129-
.filter((req) => req.url?.includes('/red_team_scans'))
131+
.filter((req) => req.url?.includes('/red_team_scans') || req.url?.includes('/hidden/profiles'))
130132
.map((req) => ({
131133
method: req.method,
132134
path: req.url?.split('?')[0],
133135
}));
134136

135137
expect(redteamRequests).toEqual([
138+
{ method: 'GET', path: '/api/hidden/profiles' }, // resolve default profile
136139
{ method: 'POST', path: scanPath }, // create scan
137140
{ method: 'POST', path: `${scanPath}/${scanId}/next` }, // get prompts (returns 1 chat)
138141
{ method: 'GET', path: `${scanPath}/${scanId}/status` }, // progress update
139142
{ method: 'POST', path: `${scanPath}/${scanId}/next` }, // get prompts (returns empty, loop ends)
140143
{ method: 'GET', path: `${scanPath}/${scanId}/status` }, // final progress
141-
{ method: 'GET', path: `${scanPath}/${scanId}` }, // fetch result
144+
{ method: 'GET', path: `${scanPath}/${scanId}/report` }, // fetch report
142145
]);
143146

144147
expect(report).toMatchObject({
@@ -430,4 +433,80 @@ describe('snyk redteam (mocked servers only)', () => {
430433
expect(result).toHaveExitCode(2);
431434
expect(result.stdout).toContain('not a valid UUID');
432435
});
436+
437+
test('`redteam --list-profiles` lists available profiles', async () => {
438+
const result = await runSnykCLI(
439+
`redteam --experimental --list-profiles`,
440+
{
441+
env,
442+
},
443+
);
444+
expect(result).toHaveExitCode(0);
445+
expect(result.stdout).toContain('Available profiles');
446+
expect(result.stdout).toContain('fast');
447+
expect(result.stdout).toContain('security');
448+
});
449+
450+
test('`redteam --list-goals` lists available goals', async () => {
451+
const result = await runSnykCLI(
452+
`redteam --experimental --list-goals`,
453+
{
454+
env,
455+
},
456+
);
457+
expect(result).toHaveExitCode(0);
458+
expect(result.stdout).toContain('Available goals');
459+
expect(result.stdout).toContain('system_prompt_extraction');
460+
expect(result.stdout).toContain('prompt_injection');
461+
});
462+
463+
test('`redteam --goals` runs scan with specified goals', async () => {
464+
const result = await runSnykCLI(
465+
`redteam --config=${redteamTarget} --experimental --tenant-id=${tenantId} --target-url=${targetURL} --goals=system_prompt_extraction`,
466+
{
467+
env,
468+
},
469+
);
470+
expect(result).toHaveExitCode(0);
471+
472+
let report: any;
473+
expect(() => {
474+
report = JSON.parse(extractJSON(result.stdout));
475+
}).not.toThrow();
476+
expect(report.id).toBeDefined();
477+
expect(report.results).toBeDefined();
478+
479+
const requests = server
480+
.getRequests()
481+
.filter((req) => req.url?.includes('/hidden/profiles'));
482+
expect(requests).toHaveLength(0);
483+
});
484+
485+
test('`redteam --profile` runs scan with specified profile', async () => {
486+
const result = await runSnykCLI(
487+
`redteam --config=${redteamTarget} --experimental --tenant-id=${tenantId} --target-url=${targetURL} --profile=security`,
488+
{
489+
env,
490+
},
491+
);
492+
expect(result).toHaveExitCode(0);
493+
494+
let report: any;
495+
expect(() => {
496+
report = JSON.parse(extractJSON(result.stdout));
497+
}).not.toThrow();
498+
expect(report.id).toBeDefined();
499+
expect(report.results).toBeDefined();
500+
});
501+
502+
test('`redteam --goals --profile` fails with both flags', async () => {
503+
const result = await runSnykCLI(
504+
`redteam --config=${redteamTarget} --experimental --tenant-id=${tenantId} --target-url=${targetURL} --goals=system_prompt_extraction --profile=fast`,
505+
{
506+
env,
507+
},
508+
);
509+
expect(result).toHaveExitCode(2);
510+
expect(result.stdout).toContain('cannot be used together');
511+
});
433512
});

0 commit comments

Comments
 (0)