[Cases] Adding mime type telemetry

[jonathan-buttner]

This PR adds telemetry for collecting statistics for the mimetypes of the files uploaded to cases.

It only looks for the top 20 mime types using a terms aggregation. It does this across all cases and broken down by owner. On the telemetry server we'll need to create a single document per entry in the array here so we can do filtering for the visualizations.

Example

  "topMimeTypes": [
    {
      "count": 2,
      "name": "image/png"
    },
    {
      "count": 1,
      "name": "application/json"
    },
    {
      "count": 1,
      "name": "text/plain"
    }
  ]

Example telemetry result

                     "cases": {
                        "cases": {
                            "all": {
                                "total": 1,
                                "daily": 1,
                                "weekly": 1,
                                "monthly": 1,
                                "status": {
                                    "open": 1,
                                    "inProgress": 0,
                                    "closed": 0
                                },
                                "syncAlertsOn": 1,
                                "syncAlertsOff": 0,
                                "totalUsers": 1,
                                "totalParticipants": 1,
                                "totalTags": 1,
                                "totalWithAlerts": 0,
                                "totalWithConnectors": 0,
                                "latestDates": {
                                    "createdAt": "2023-04-11T16:07:00.565Z",
                                    "updatedAt": "2023-04-11T16:07:17.712Z",
                                    "closedAt": ""
                                },
                                "assignees": {
                                    "total": 0,
                                    "totalWithZero": 1,
                                    "totalWithAtLeastOne": 0
                                },
                                "attachmentFramework": {
                                    "externalAttachments": [
                                        {
                                            "type": ".files",
                                            "average": 4,
                                            "maxOnACase": 4,
                                            "total": 4
                                        }
                                    ],
                                    "persistableAttachments": [],
                                    "files": {
                                        "averageSize": 3,
                                        "average": 4,
                                        "maxOnACase": 4,
                                        "total": 4,
                                        "topMimeTypes": [
                                            {
                                                "count": 2,
                                                "name": "image/png"
                                            },
                                            {
                                                "count": 1,
                                                "name": "application/json"
                                            },
                                            {
                                                "count": 1,
                                                "name": "text/plain"
                                            }
                                        ]
                                    }
                                }
                            },
                            "sec": {
                                "total": 0,
                                "daily": 0,
                                "weekly": 0,
                                "monthly": 0,
                                "attachmentFramework": {
                                    "externalAttachments": [],
                                    "persistableAttachments": [],
                                    "files": {
                                        "averageSize": 0,
                                        "average": 0,
                                        "maxOnACase": 0,
                                        "total": 0,
                                        "topMimeTypes": []
                                    }
                                },
                                "assignees": {
                                    "total": 0,
                                    "totalWithZero": 0,
                                    "totalWithAtLeastOne": 0
                                }
                            },
                            "obs": {
                                "total": 0,
                                "daily": 0,
                                "weekly": 0,
                                "monthly": 0,
                                "attachmentFramework": {
                                    "externalAttachments": [],
                                    "persistableAttachments": [],
                                    "files": {
                                        "averageSize": 0,
                                        "average": 0,
                                        "maxOnACase": 0,
                                        "total": 0,
                                        "topMimeTypes": []
                                    }
                                },
                                "assignees": {
                                    "total": 0,
                                    "totalWithZero": 0,
                                    "totalWithAtLeastOne": 0
                                }
                            },
                            "main": {
                                "total": 1,
                                "daily": 1,
                                "weekly": 1,
                                "monthly": 1,
                                "attachmentFramework": {
                                    "externalAttachments": [
                                        {
                                            "type": ".files",
                                            "average": 4,
                                            "maxOnACase": 4,
                                            "total": 4
                                        }
                                    ],
                                    "persistableAttachments": [],
                                    "files": {
                                        "averageSize": 3,
                                        "average": 4,
                                        "maxOnACase": 4,
                                        "total": 4,
                                        "topMimeTypes": [
                                            {
                                                "count": 2,
                                                "name": "image/png"
                                            },
                                            {
                                                "count": 1,
                                                "name": "application/json"
                                            },
                                            {
                                                "count": 1,
                                                "name": "text/plain"
                                            }
                                        ]
                                    }
                                },
                                "assignees": {
                                    "total": 0,
                                    "totalWithZero": 1,
                                    "totalWithAtLeastOne": 0
                                }
                            }
                        },
                        "userActions": {
                            "all": {
                                "total": 5,
                                "daily": 5,
                                "weekly": 5,
                                "monthly": 5,
                                "maxOnACase": 5
                            }
                        },
                        "comments": {
                            "all": {
                                "total": 0,
                                "daily": 0,
                                "weekly": 0,
                                "monthly": 0,
                                "maxOnACase": 0
                            }
                        },
                        "alerts": {
                            "all": {
                                "total": 0,
                                "daily": 0,
                                "weekly": 0,
                                "monthly": 0,
                                "maxOnACase": 0
                            }
                        },
                        "connectors": {
                            "all": {
                                "all": {
                                    "totalAttached": 0
                                },
                                "itsm": {
                                    "totalAttached": 0
                                },
                                "sir": {
                                    "totalAttached": 0
                                },
                                "jira": {
                                    "totalAttached": 0
                                },
                                "resilient": {
                                    "totalAttached": 0
                                },
                                "swimlane": {
                                    "totalAttached": 0
                                },
                                "maxAttachedToACase": 0
                            }
                        },
                        "pushes": {
                            "all": {
                                "total": 0,
                                "maxOnACase": 0
                            }
                        },
                        "configuration": {
                            "all": {
                                "closure": {
                                    "manually": 0,
                                    "automatic": 0
                                }
                            }
                        }
                    }

Testing

To test modify this file: https://github.com/elastic/kibana/blob/main/x-pack/plugins/cases/server/telemetry/schedule_telemetry_task.ts

With:

export const scheduleCasesTelemetryTask = (
  taskManager: TaskManagerStartContract,
  logger: Logger
) => {
  (async () => {
    await taskManager
      .ensureScheduled({
        id: CASES_TELEMETRY_TASK_NAME,
        taskType: CASES_TELEMETRY_TASK_NAME,
        schedule: {
          interval: `${MINUTES_ON_HALF_DAY}m`,
        },
        scope: ['cases'],
        params: {},
        state: {},
      })
      .catch((err) => {
        logger.debug(
          `Error scheduling cases task with ID ${CASES_TELEMETRY_TASK_NAME} and type ${CASES_TELEMETRY_TASK_NAME}. Received ${err.message}`
        );
      });

    await taskManager.runSoon(CASES_TELEMETRY_TASK_NAME);
  })();
};

This will cause the telemetry to be sent as soon as the server is restarted.

To generate files and attachments to add stats to the telemetry I created this python script: https://github.com/elastic/cases-files-generator

To retrieve the telemetry:

POST http://localhost:5601/api/telemetry/v2/clusters/_stats
{
   "refreshCache": true,
   "unencrypted": true
}

[jonathan-buttner]

This was a bug before. We're doing an average aggregation so we need to use value to get the actual result.

[kibana-ci]

💚 Build Succeeded

• Buildkite Build
• Commit: 1e2a315

Metrics [docs]

Unknown metric groups

ESLint disabled line counts

id	before	after	diff
securitySolution	433	436	+3

Total ESLint disabled count

id	before	after	diff
securitySolution	513	516	+3

History

• 💔 Build #119053 failed a351747
• 💔 Build #119030 failed 31801a7
• 💔 Build #118794 failed 97bed79
• 💔 Build #118743 failed f24f4be

To update your PR or re-run it, just comment with:
@elasticmachine merge upstream

[elasticmachine]

Pinging @elastic/response-ops (Team:ResponseOps)

[elasticmachine]

Pinging @elastic/response-ops-cases (Feature:Cases)

[afharo]

cc @elastic/platform-analytics @shahinakmal @elasticelvin