Merge pull request #4 from guardrails-ai/get-validator-metrics-summary

Get validator metrics
guardrails-ai · Nov 7, 2023 · 73094ea · 73094ea
2 parents 5ab8c35 + 2f6603d
commit 73094ea
Show file tree

Hide file tree

Showing 2 changed files with 120 additions and 3 deletions.
diff --git a/service-specs/guardrails-service-spec.yml b/service-specs/guardrails-service-spec.yml
@@ -307,11 +307,15 @@ components:
       type: object
       properties:
         schema:
-          type: object
-          additionalProperties:
-            $ref: "#/components/schemas/DataType"
+          oneOf:
+            - $ref: "#/components/schemas/JsonSchema"
+            - $ref: "#/components/schemas/DataType"
       required:
         - schema
+    JsonSchema:
+      type: object
+      additionalProperties:
+        $ref: "#/components/schemas/DataType"
     DataType:
       type: object
       properties:

diff --git a/service-specs/reporting-service-spec.yml b/service-specs/reporting-service-spec.yml
@@ -79,6 +79,62 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/HttpError"
+  /validators/metrics:
+    get:
+      operationId: getValidatorMetrics
+      summary: Returns an array of ValidatorMetrics for a given time range.  If startTime or endTime are ommitted, a five minute duration is used by default.  Default page size is 10, but this page size applied to guard steps not validator metrics. Have to think this one out a little more.
+      parameters:
+        - $ref: "#/components/parameters/StartTime"
+        - $ref: "#/components/parameters/EndTime"
+        - $ref: "#/components/parameters/PageNumber"
+      tags:
+        - Metrics
+      security:
+        - apiKeyAuth: []
+        - bearerAuth: []
+      responses:
+        "200":
+          description: Returns metrics for Guards
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ValidatorMetrics"
+        default:
+          description: Unexpected error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/HttpError"
+  /validators/{validatorName}/runs/{requestId}/instances/{validatorInstanceId}/metrics:
+    get:
+      operationId: getValidatorRunMetrics
+      summary: Returns an arary of ValidatorRunMetrics for a given request.
+      parameters:
+        - $ref: "#/components/parameters/ValidatorName"
+        - $ref: "#/components/parameters/RequestId"
+        - $ref: "#/components/parameters/ValidatorInstanceId"
+      tags:
+        - Metrics
+      security:
+        - apiKeyAuth: []
+        - bearerAuth: []
+      responses:
+        "200":
+          description: Returns metrics for Guard request
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ValidatorRunMetrics"
+        default:
+          description: Unexpected error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/HttpError"
 components:
   securitySchemes:
     bearerAuth:
@@ -206,8 +262,14 @@ components:
     ValidatorMetrics:
       type: object
       properties:
+        instanceId:
+          type: string
         name:
           type: string
+        requestId:
+          type: string
+        guardName:
+          type: string
         startTime:
           type: string
           format: date-time
@@ -226,13 +288,50 @@ components:
           type: number
           description: The number of tokens consumed if the validator calls an LLM. Only applies to validators that call LLMs.
       required:
+        - instanceId
         - name
+        - requestId
+        - guardName
         - startTime
         - endTime
         - durationInMillis
         - input
         - output
         - status
+    ValidatorRunMetrics:
+      allOf:
+        - $ref: '#/components/schemas/ValidatorMetrics'
+        - $ref: '#/components/schemas/ValidatorRunMetricDetails'
+    ValidatorRunMetricDetails:
+      type: object
+      properties:
+        stepNumber:
+          type: integer
+        resultType:
+          type: string
+        failAction:
+          $ref: '#/components/schemas/OnFailOptions'
+        parameters:
+          type: object
+          properties: {}
+          additionalProperties:
+            type: string
+      required:
+        - stepNumber
+        - resultType
+        - failAction
+        - parameters
+    OnFailOptions:
+      type: string
+      enum: [
+        "exception",
+        "filter",
+        "fix",
+        "fix_reask",
+        "noop",
+        "reask",
+        "refrain"
+      ]
   parameters:
     StartTime:
       name: startTime
@@ -270,5 +369,19 @@ components:
       in: path
       description: Request id aka trace id for a validate-{guard-name} span
       required: true
+      schema:
+        type: string
+    ValidatorName:
+      name: validatorName
+      in: path
+      description: the validator's registered name
+      required: true
+      schema:
+        type: string
+    ValidatorInstanceId:
+      name: validatorInstanceId
+      in: path
+      description: the instanceId from ValidatorMetrics
+      required: true
       schema:
         type: string