Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enhance JSON Schema and JSON Keys plugins with improved error handling and explanations #586

Merged
merged 3 commits into from
Sep 10, 2024
Merged

Enhance JSON Schema and JSON Keys plugins with improved error handling and explanations #586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b1e1fbc
Fixed JSON schema and JSON keys plugins for different json outputs th…
roh26it Sep 8, 2024
5e6cf0c
formatting
roh26it Sep 8, 2024
5577ee1
chore: remove .only from json keys test
VisargD Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 176 additions & 19 deletions plugins/default/default.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ describe('jsonSchema handler', () => {
it('should validate JSON in response text', async () => {
const context: PluginContext = {
response: {
text: `adding some text before this \`\`\`json\n{"key1": "value"}\n\`\`\`\n and adding some text after {"key":"value"}`,
text: `adding some text before this \`\`\`json\n{"key": "value"}\n\`\`\`\n and adding some text after {"key":"value"}`,
},
};
const eventType = 'afterRequestHook';
Expand All @@ -27,7 +27,51 @@ describe('jsonSchema handler', () => {

expect(result.error).toBe(null);
expect(result.verdict).toBe(true);
expect(result.data).toEqual({ key: 'value' });
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toEqual({ key: 'value' });
expect(result.data.explanation).toContain('Successfully validated');
});

it('should validate JSON in response text - complex', async () => {
const context: PluginContext = {
response: {
text: '```json\n{\n "title": "The Rise of AI Agents: Transforming the Future",\n "short_intro": "Artificial Intelligence (AI) agents are revolutionizing various sectors, from healthcare to finance. In this blog, we explore the development of AI agents, their applications, and their potential to reshape our world."\n}\n```',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
schema: z.object({ title: z.string(), short_intro: z.string() }),
};

const result = await jsonSchemaHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(true);
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toHaveProperty('title');
expect(result.data.matchedJson).toHaveProperty('short_intro');
expect(result.data.explanation).toContain('Successfully validated');
});

it('should validate only JSON in response text', async () => {
const context: PluginContext = {
response: {
text: '{\n "title": "The Rise of AI Agents: Transforming the Future",\n "short_intro": "Artificial Intelligence (AI) agents are revolutionizing various sectors, from healthcare to finance. In this blog, we explore the development of AI agents, their applications, and their potential to reshape our world."\n}',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
schema: z.object({ title: z.string(), short_intro: z.string() }),
};

const result = await jsonSchemaHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(true);
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toHaveProperty('title');
expect(result.data.matchedJson).toHaveProperty('short_intro');
expect(result.data.explanation).toContain('Successfully validated');
});

it('should return a false verdict for invalid JSON in response text', async () => {
Expand All @@ -45,39 +89,63 @@ describe('jsonSchema handler', () => {

expect(result.error).toBe(null);
expect(result.verdict).toBe(false);
expect(result.data).toBe(null);
expect(result.data).toBeDefined();
expect(result.data.explanation).toContain('Failed to validate');
expect(result.data.validationErrors).toBeDefined();
expect(Array.isArray(result.data.validationErrors)).toBe(true);
});

it('should return explanation when no valid JSON is found', async () => {
const context: PluginContext = {
response: {
text: 'This is just plain text with no JSON',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
schema: z.object({ key: z.string() }),
};

const result = await jsonSchemaHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(false);
expect(result.data).toBeDefined();
expect(result.data.explanation).toContain('No valid JSON found');
});
});

describe('jsonKeys handler', () => {
it('should return true verdict for any key in JSON', async () => {
it('should validate JSON with "any" operator', async () => {
const context: PluginContext = {
response: {
text: `adding some text before this \`\`\`json\n{"key1": "value"}\n\`\`\`\n and adding some text after {"key":"value"}`,
text: '{"key1": "value1", "key2": "value2"}',
},
};
const eventType = 'afterRequestHook';

const parameters: PluginParameters = {
keys: ['key1'],
keys: ['key1', 'key3'],
operator: 'any',
};

const result = await jsonKeysHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(true);
expect(result.data).toEqual({ key1: 'value' });
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toEqual({ key1: 'value1', key2: 'value2' });
expect(result.data.explanation).toContain('Successfully matched');
expect(result.data.presentKeys).toContain('key1');
expect(result.data.missingKeys).toContain('key3');
});

it('should return false verdict for all keys in JSON', async () => {
it('should validate JSON with "all" operator', async () => {
const context: PluginContext = {
response: {
text: `adding some text before this \`\`\`json\n{"key1": "value"}\n\`\`\`\n and adding some text after {"key":"value"}`,
text: '{"key1": "value1", "key2": "value2"}',
},
};
const eventType = 'afterRequestHook';

const parameters: PluginParameters = {
keys: ['key1', 'key2'],
operator: 'all',
Expand All @@ -86,31 +154,120 @@ describe('jsonKeys handler', () => {
const result = await jsonKeysHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(false);
expect(result.verdict).toBe(true);
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toEqual({ key1: 'value1', key2: 'value2' });
expect(result.data.explanation).toContain('Successfully matched');
expect(result.data.presentKeys).toEqual(['key1', 'key2']);
expect(result.data.missingKeys).toEqual([]);
});

// console.log(result);
it('should validate JSON with "none" operator', async () => {
const context: PluginContext = {
response: {
text: '{"key1": "value1", "key2": "value2"}',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
keys: ['key3', 'key4'],
operator: 'none',
};

const result = await jsonKeysHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(true);
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toEqual({ key1: 'value1', key2: 'value2' });
expect(result.data.explanation).toContain('Successfully matched');
expect(result.data.presentKeys).toEqual([]);
expect(result.data.missingKeys).toEqual(['key3', 'key4']);
});

it('should return true verdict for none of the keys in JSON', async () => {
it('should handle JSON in code blocks', async () => {
const context: PluginContext = {
response: {
text: `adding some text before this \`\`\`json\n{"key1": "value"}\n\`\`\`\n and adding some text after {"key":"value"}`,
text: '```json\n{"key1": "value1", "key2": "value2"}\n```',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
keys: ['key1', 'key2'],
operator: 'all',
};

const result = await jsonKeysHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(true);
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toEqual({ key1: 'value1', key2: 'value2' });
expect(result.data.explanation).toContain('Successfully matched');
});

it('should return false verdict when keys are not found', async () => {
const context: PluginContext = {
response: {
text: '{"key1": "value1", "key2": "value2"}',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
keys: ['key2'],
operator: 'none',
keys: ['key3', 'key4'],
operator: 'any',
};

const result = await jsonKeysHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(false);
expect(result.data).toBeDefined();
expect(result.data.explanation).toContain('Failed to match');
expect(result.data.presentKeys).toEqual([]);
expect(result.data.missingKeys).toEqual(['key3', 'key4']);
});

it('should handle multiple JSON objects in text', async () => {
const context: PluginContext = {
response: {
text: '{"key1": "value1"} Some text {"key2": "value2", "key3": "value3"}',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
keys: ['key2', 'key3'],
operator: 'all',
};

const result = await jsonKeysHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(true);
expect(result.data).toEqual({ key1: 'value' });
expect(result.data).toBeDefined();
expect(result.data.matchedJson).toEqual({ key2: 'value2', key3: 'value3' });
expect(result.data.explanation).toContain('Successfully matched');
});

// console.log(result);
it('should return explanation when no valid JSON is found', async () => {
const context: PluginContext = {
response: {
text: 'This is just plain text with no JSON',
},
};
const eventType = 'afterRequestHook';
const parameters: PluginParameters = {
keys: ['key1', 'key2'],
operator: 'any',
};

const result = await jsonKeysHandler(context, parameters, eventType);

expect(result.error).toBe(null);
expect(result.verdict).toBe(false);
expect(result.data).toBeDefined();
expect(result.data.explanation).toContain('No valid JSON found');
expect(result.data.operator).toBe('any');
});
});

Expand Down
74 changes: 58 additions & 16 deletions plugins/default/jsonKeys.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,36 @@ export const handler: PluginHandler = async (
) => {
let error = null;
let verdict = false;
let data = null;
let data: any = null;

try {
const keys = parameters.keys;
const operator = parameters.operator;
let responseText = getText(context, eventType);

const jsonRegex = /{.*?}/g;
const jsonMatches = responseText.match(jsonRegex);
// Extract JSON from code blocks and general text
const extractJson = (text: string): string[] => {
const codeBlockRegex = /```+(?:json)?\s*([\s\S]*?)```+/g;
const jsonRegex = /{[\s\S]*?}/g;
const matches = [];

if (jsonMatches) {
// Extract from code blocks
let match;
while ((match = codeBlockRegex.exec(text)) !== null) {
matches.push(match[1].trim());
}

// Extract JSON-like structures
while ((match = jsonRegex.exec(text)) !== null) {
matches.push(match[0]);
}

return matches;
};

const jsonMatches = extractJson(responseText);

if (jsonMatches.length > 0) {
for (const jsonMatch of jsonMatches) {
let responseJson: any;
try {
Expand All @@ -34,33 +53,56 @@ export const handler: PluginHandler = async (

responseJson = responseJson || {};

const presentKeys = keys.filter((key: string) =>
responseJson.hasOwnProperty(key)
);
const missingKeys = keys.filter(
(key: string) => !responseJson.hasOwnProperty(key)
);

// Check if the JSON contains any, all or none of the keys
switch (operator) {
case 'any':
verdict = keys.some((key: string) =>
responseJson.hasOwnProperty(key)
);
verdict = presentKeys.length > 0;
break;
case 'all':
verdict = keys.every((key: string) =>
responseJson.hasOwnProperty(key)
);
verdict = missingKeys.length === 0;
break;
case 'none':
verdict = keys.every(
(key: string) => !responseJson.hasOwnProperty(key)
);
verdict = presentKeys.length === 0;
break;
}

if (verdict) {
data = responseJson;
data = {
matchedJson: responseJson,
explanation: `Successfully matched JSON with '${operator}' keys criteria.`,
presentKeys,
missingKeys,
};
break;
} else {
data = {
matchedJson: responseJson,
explanation: `Failed to match JSON with '${operator}' keys criteria.`,
presentKeys,
missingKeys,
};
}
}
} else {
data = {
explanation: 'No valid JSON found in the response.',
requiredKeys: keys,
operator,
};
}
} catch (e) {
error = e as Error;
} catch (e: any) {
error = e;
data = {
explanation: 'An error occurred while processing the JSON.',
error: e.message,
};
}

return { error, verdict, data };
Expand Down
Loading
Loading