From 6cc9aa614bce087496b8680d4f48c50f3a6373dd Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 12 Oct 2023 13:48:23 -0500 Subject: [PATCH 1/4] fix string schema support --- service-specs/guardrails-service-spec.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/service-specs/guardrails-service-spec.yml b/service-specs/guardrails-service-spec.yml index 5b4d5ff..63a4b3f 100644 --- a/service-specs/guardrails-service-spec.yml +++ b/service-specs/guardrails-service-spec.yml @@ -307,11 +307,15 @@ components: type: object properties: schema: - type: object - additionalProperties: - $ref: "#/components/schemas/DataType" + oneOf: + - $ref: "#/components/schemas/JsonSchema" + - $ref: "#/components/schemas/DataType" required: - schema + JsonSchema: + type: object + additionalProperties: + $ref: "#/components/schemas/DataType" DataType: type: object properties: From c2e7fe96ea8931d1976dd4593a4415217769254c Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 12 Oct 2023 15:20:14 -0500 Subject: [PATCH 2/4] /validators/metrics endpoint --- service-specs/reporting-service-spec.yml | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/service-specs/reporting-service-spec.yml b/service-specs/reporting-service-spec.yml index 7de30e2..1935b09 100644 --- a/service-specs/reporting-service-spec.yml +++ b/service-specs/reporting-service-spec.yml @@ -79,6 +79,34 @@ paths: application/json: schema: $ref: "#/components/schemas/HttpError" + /validators/metrics: + get: + operationId: getValidatorMetrics + summary: Returns an array of ValidatorMetrics for a given time range. If startTime or endTime are ommitted, a five minute duration is used by default. Default page size is 10. + parameters: + - $ref: "#/components/parameters/StartTime" + - $ref: "#/components/parameters/EndTime" + - $ref: "#/components/parameters/PageNumber" + tags: + - Metrics + security: + - apiKeyAuth: [] + - bearerAuth: [] + responses: + "200": + description: Returns metrics for Guards + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ValidatorMetrics" + default: + description: Unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/HttpError" components: securitySchemes: bearerAuth: @@ -208,6 +236,10 @@ components: properties: name: type: string + requestId: + type: string + guardName: + type: string startTime: type: string format: date-time @@ -227,6 +259,8 @@ components: description: The number of tokens consumed if the validator calls an LLM. Only applies to validators that call LLMs. required: - name + - requestId + - guardName - startTime - endTime - durationInMillis From c79dec92aceaf9dcd19f26f7c211075aa081132a Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Fri, 13 Oct 2023 13:03:23 -0500 Subject: [PATCH 3/4] start validator metrics details --- service-specs/reporting-service-spec.yml | 79 ++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/service-specs/reporting-service-spec.yml b/service-specs/reporting-service-spec.yml index 1935b09..48af687 100644 --- a/service-specs/reporting-service-spec.yml +++ b/service-specs/reporting-service-spec.yml @@ -107,6 +107,34 @@ paths: application/json: schema: $ref: "#/components/schemas/HttpError" + /validators/{validatorName}/runs/{requestId}/instances/{validatorInstanceId}/metrics: + get: + operationId: getValidatorRunMetrics + summary: Returns an arary of ValidatorRunMetrics for a given request. + parameters: + - $ref: "#/components/parameters/ValidatorInstanceId" + - $ref: "#/components/parameters/ValidatorName" + - $ref: "#/components/parameters/RequestId" + tags: + - Metrics + security: + - apiKeyAuth: [] + - bearerAuth: [] + responses: + "200": + description: Returns metrics for Guard request + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/ValidatorRunMetrics" + default: + description: Unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/HttpError" components: securitySchemes: bearerAuth: @@ -234,6 +262,8 @@ components: ValidatorMetrics: type: object properties: + instanceId: + type: string name: type: string requestId: @@ -258,6 +288,7 @@ components: type: number description: The number of tokens consumed if the validator calls an LLM. Only applies to validators that call LLMs. required: + - instanceId - name - requestId - guardName @@ -267,6 +298,40 @@ components: - input - output - status + ValidatorRunMetrics: + allOf: + - $ref: '#/components/schemas/ValidatorMetrics' + - $ref: '#/components/schemas/ValidatorRunMetricDetails' + ValidatorRunMetricDetails: + type: object + properties: + stepNumber: + type: integer + resultType: + type: string + failAction: + $ref: '#/components/schemas/OnFailOptions' + parameters: + type: object + properties: {} + additionalProperties: + type: string + required: + - stepNumber + - resultType + - failAction + - parameters + OnFailOptions: + type: string + enum: [ + "exception", + "filter", + "fix", + "fix_reask", + "noop", + "reask", + "refrain" + ] parameters: StartTime: name: startTime @@ -304,5 +369,19 @@ components: in: path description: Request id aka trace id for a validate-{guard-name} span required: true + schema: + type: string + ValidatorName: + name: validatorName + in: path + description: the validator's registered name + required: true + schema: + type: string + ValidatorInstanceId: + name: validatorInstanceId + in: path + description: the instanceId from ValidatorMetrics + required: true schema: type: string \ No newline at end of file From 2f6603d1bfc6813bef5791549c90a3db6659592d Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Fri, 13 Oct 2023 15:45:50 -0500 Subject: [PATCH 4/4] note about poor pagination --- service-specs/reporting-service-spec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service-specs/reporting-service-spec.yml b/service-specs/reporting-service-spec.yml index 48af687..453eeb4 100644 --- a/service-specs/reporting-service-spec.yml +++ b/service-specs/reporting-service-spec.yml @@ -82,7 +82,7 @@ paths: /validators/metrics: get: operationId: getValidatorMetrics - summary: Returns an array of ValidatorMetrics for a given time range. If startTime or endTime are ommitted, a five minute duration is used by default. Default page size is 10. + summary: Returns an array of ValidatorMetrics for a given time range. If startTime or endTime are ommitted, a five minute duration is used by default. Default page size is 10, but this page size applied to guard steps not validator metrics. Have to think this one out a little more. parameters: - $ref: "#/components/parameters/StartTime" - $ref: "#/components/parameters/EndTime" @@ -112,9 +112,9 @@ paths: operationId: getValidatorRunMetrics summary: Returns an arary of ValidatorRunMetrics for a given request. parameters: - - $ref: "#/components/parameters/ValidatorInstanceId" - $ref: "#/components/parameters/ValidatorName" - $ref: "#/components/parameters/RequestId" + - $ref: "#/components/parameters/ValidatorInstanceId" tags: - Metrics security: