mistk-evaluation-api.yaml

---
swagger: "2.0"
info:
  version: "1.0.0"
  title: "Model Integration Software ToolKit - Metric Evaluation"
basePath: "/v1/mistk/evaluation"
schemes:
- "http"
consumes:
- "application/json"
produces:
- "application/json"
paths:
  /evaluate:
    post:
      tags:
      - "Evaluation Plugin Endpoint"
      summary: "Performs the evaluation defined for this plugin"
      operationId: "evaluate"
      parameters:
      - in: "body"
        name: "initParams"
        description: "A list of metrics to run and ground truth and prediction file\
          \ paths to run the metrics against"
        required: true
        schema:
          $ref: "#/definitions/EvaluationSpecificationInitParams"
      responses:
        200:
          description: "Executing evaluation"
        405:
          description: "Invalid input"
          schema:
            $ref: "#/definitions/ServiceError"
        500:
          description: "Unexpected error"
          schema:
            $ref: "#/definitions/ServiceError"
      x-swagger-router-controller: "mistk.evaluation.service"
  /metrics:
    get:
      tags:
      - "Evaluation Plugin Endpoint"
      summary: "Retrieves the metrics available to perform for the evaluation plugin"
      operationId: "get_metrics"
      parameters: []
      responses:
        200:
          description: "Metrics able to be run for evaluation"
          schema:
            type: "array"
            items:
              $ref: "#/definitions/MistkMetric"
        405:
          description: "Invalid input"
          schema:
            $ref: "#/definitions/ServiceError"
        500:
          description: "Unexpected error"
          schema:
            $ref: "#/definitions/ServiceError"
      x-swagger-router-controller: "mistk.evaluation.service"
  /status:
    get:
      tags:
      - "Evaluation Plugin Endpoint"
      summary: "Retrieves the status of the evaluation plugin"
      operationId: "get_status"
      parameters:
      - name: "watch"
        in: "query"
        description: "Watch for changes to the described resources and return them\
          \ as a stream of add, update, and remove notifications. Specify resourceVersion.\n"
        required: false
        type: "boolean"
      - name: "resourceVersion"
        in: "query"
        description: "When specified with a watch call, shows changes that occur after\
          \ that particular version of a resource. Defaults to changes from the beginning\
          \ of history.\n"
        required: false
        type: "number"
      responses:
        200:
          description: "The status of the Evaluation Plugin"
          schema:
            $ref: "#/definitions/EvaluationInstanceStatus"
        500:
          description: "Unexpected error"
          schema:
            $ref: "#/definitions/ServiceError"
      x-swagger-router-controller: "mistk.evaluation.service"
  /shutdown:
    post:
      tags:
      - "Evaluation Plugin Endpoint"
      summary: "Shutdowns the evaluation plugin and cleans up any resources."
      operationId: "terminate"
      parameters: []
      responses:
        200:
          description: "Evaluation Plugin successfully shutdown"
        500:
          description: "Unexpected error"
          schema:
            $ref: "#/definitions/ServiceError"
      x-swagger-router-controller: "mistk.evaluation.service"
  /apiVersion:
    get:
      tags:
      - "Evaluation Plugin Endpoint"
      summary: "Returns the version of the MISTK API"
      description: "Returns the version of the MISTK API"
      operationId: "get_api_version"
      parameters: []
      responses:
        200:
          description: "The MISTK API Version\n"
        500:
          description: "General server runtime exception"
      x-swagger-router-controller: "mistk.evaluation.service"
definitions:
  EvaluationSpecificationInitParams:
    type: "object"
    required:
    - "assessment_type"
    - "evaluation_input_format"
    - "ground_truth_path"
    - "input_data_path"
    - "metrics"
    properties:
      assessment_type:
        type: "string"
        description: "Assessment type to use for the evaluation"
      metrics:
        type: "array"
        description: "A list of metrics to use for the evaluation"
        items:
          $ref: "#/definitions/MistkMetric"
      input_data_path:
        type: "string"
        description: "Path to input data for the evaluation"
      evaluation_input_format:
        type: "string"
        description: "The format of the input data"
        enum:
        - "predictions"
        - "generations"
      ground_truth_path:
        type: "string"
        description: "Path to ground_truth.csv file"
      evaluation_path:
        type: "string"
        description: "Path for evaluation output file"
      properties:
        type: "object"
        description: "A dictionary of key value pairs for evaluation plugin arguments."
        properties: {}
    description: "The initialization parameters used by the Evaluation Specification\
      \ when calling its initialize function\n"
    example:
      ground_truth_path: "ground_truth_path"
      input_data_path: "input_data_path"
      evaluation_path: "evaluation_path"
      metrics:
      - ""
      - ""
      evaluation_input_format: "predictions"
      assessment_type: "assessment_type"
      properties: "{}"
  ServiceError:
    type: "object"
    required:
    - "message"
    properties:
      code:
        type: "integer"
        format: "int32"
      message:
        type: "string"
    description: "An custom error class used by component services"
  MistkMetric:
    allOf:
    - $ref: "#/definitions/Metric"
  Metric:
    type: "object"
    required:
    - "objectInfo"
    properties:
      objectInfo:
        $ref: "#/definitions/ObjectInfo"
      implementationReference:
        $ref: "#/definitions/ObjectReference"
      package:
        type: "string"
        description: "The name of the package containing the implementation of this\
          \ metric.\n"
      method:
        type: "string"
        description: "The name of the method to be called when applying the metric.\n"
      defaultArgs:
        type: "object"
        description: "The default arguments passed to the method when the metric is\
          \ called.  These can be overwritten when the metric is associated with an\
          \ assessment.\n"
        properties: {}
      dataParameters:
        $ref: "#/definitions/MetricDataParameters"
      assessmentTypes:
        type: "array"
        description: "The types of assessments this metric can be used for.\n"
        items:
          type: "string"
      version:
        type: "string"
        description: "The version of this metric"
    description: "Evaluation criteria for measuring the performance of an algorithm’\
      s implementation.\n"
    x-sml-resource: "Metric"
    x-smlclient-verbs:
    - "create"
    - "read"
    - "update"
    - "delete"
    - "list"
    x-sml-scope: "project"
  ObjectInfo:
    type: "object"
    properties:
      kind:
        type: "string"
        description: "This is the type of the resource (e.g., Model, Dataset). The\
          \ field is primarily used when returning watch events, where the type of\
          \ the field will be unknown.  When submitting resources, this field will\
          \ often simply be inferred from the context of the submission.\n"
      id:
        type: "string"
        format: "uuid"
        description: "The unique identifier of the object.  This will be generated\
          \ automatically.\n"
      name:
        type: "string"
        description: "The unique name of the object. Names of SML resources should\
          \ be up to maximum length of 253 characters and consist of lower case alphanumeric\
          \ characters, -, and .\n"
      displayName:
        type: "string"
        description: "The name of this resoure, as presented via this UI.  This field\
          \ can have spaces and upper and lower case alphanumeric  characters as well\
          \ as spaces.\n"
      description:
        type: "string"
        description: "A documentation description of the resource.  This string can\
          \ be in Markdown form.\n"
      summary:
        type: "string"
        description: "A short description of this object\n"
      owner:
        type: "string"
        description: "The owner/creator of this object.\n"
      project:
        type: "string"
        description: "The project for this object (assuming the resource is project-scoped)\n"
      public:
        type: "boolean"
        description: "Boolean flag indicating whether this resource will be  accessible\
          \ to members outside of the project\n"
      labels:
        type: "object"
        description: "Key-value pairs of annotations associate with the object.\n"
        properties: {}
      resources:
        type: "array"
        description: "An array of resources/reference URIs\n"
        items:
          type: "string"
      resourceVersion:
        type: "integer"
        description: "This is the server-managed version for the object.  It is used\
          \  to support monitoring of resources.  This should not be set by the client\
          \ (and will likely be ignored if it is)\n"
      creationTime:
        type: "string"
        format: "date-time"
        description: "The date time at which this object was initially created. This\
          \ will be auto generated by the underlying database. Any user input for\
          \ this field will be ignored. This will be in the isoformat of 'YYYY-MM-DDTHH:MM:SS.ffffff'\n"
      modificationTime:
        type: "string"
        format: "date-time"
        description: "The date time at which this object was last modified. This will\
          \ be auto generated by the underlying database. Any user input for this\
          \ field will be ignored. This will be in the isoformat of 'YYYY-MM-DDTHH:MM:SS.ffffff'\n"
    description: "This is the set of fields common to all objects."
    example:
      summary: "summary"
      owner: "owner"
      creationTime: "2000-01-23T04:56:07.000+00:00"
      kind: "kind"
      displayName: "displayName"
      resourceVersion: 0
      description: "description"
      project: "project"
      resources:
      - "resources"
      - "resources"
      labels: "{}"
      public: true
      modificationTime: "2000-01-23T04:56:07.000+00:00"
      name: "name"
      id: "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
  ObjectReference:
    type: "object"
    properties:
      kind:
        type: "string"
        description: "The type of the object being referenced. This is often optional\
          \ as it will be inferrred by the context.\n"
      project:
        type: "string"
        description: "The project (namespace) of this object.\n"
      id:
        type: "string"
        format: "uuid"
        description: "The unique identifier of the object."
      name:
        type: "string"
        description: "The unique name of the object."
      instance:
        type: "object"
        description: "Optionally, the object reference can contain the complete object.\
          \  This is useful for unit testing where you don't want to go out to the\
          \ SRM to obtain the referenced object.\n"
        properties: {}
    description: "A reference to another declared object.  At least one of id or name\
      \ fields must be specified.\n"
  MetricDataParameters:
    type: "object"
    properties:
      predictionLabels:
        type: "string"
        description: "The arg name for prediction labels"
      truthLabels:
        type: "string"
        description: "The arg name for ground truth labels"
      predictionScores:
        type: "string"
        description: "The arg name for prediction scores"
      truthBounds:
        type: "string"
        description: "The arg name for ground truth bounds"
      predictionBounds:
        type: "string"
        description: "The arg name for prediction bounds"
    description: "The data parameters provides a mapping for data into the arguments\
      \ passed to the metric.  \n"
  EvaluationInstanceStatus:
    type: "object"
    required:
    - "objectInfo"
    properties:
      objectInfo:
        $ref: "#/definitions/ObjectInfo"
      state:
        type: "string"
        description: "The current state of the evaluation instance"
        enum:
        - "started"
        - "initializing"
        - "initialized"
        - "failed"
        - "ready"
        - "evaluating"
        - "completed"
      payload:
        type: "object"
        description: "Additional arbitrary information relevant to the current state.\
          \        \n"
        properties: {}
    description: "A evaluation instance status object represents the state of a running\
      \ evaluation instance. \n"
    example:
      payload: "{}"
      state: "started"
      objectInfo:
        summary: "summary"
        owner: "owner"
        creationTime: "2000-01-23T04:56:07.000+00:00"
        kind: "kind"
        displayName: "displayName"
        resourceVersion: 0
        description: "description"
        project: "project"
        resources:
        - "resources"
        - "resources"
        labels: "{}"
        public: true
        modificationTime: "2000-01-23T04:56:07.000+00:00"
        name: "name"
        id: "046b6c7f-0b8a-43b9-b35d-6489e6daee91"