Merge branch 'main' into make-metrics-internals

docs/apm/api.asciidoc

@@ -11,6 +11,7 @@ Some APM app features are provided via a REST API:
 * <<agent-config-api>>
 * <<apm-annotation-api>>
 * <<rum-sourcemap-api>>
+* <<android-sourcemap-api>>
 * <<agent-key-api>>
 
 [float]
@@ -715,6 +716,219 @@ curl -X DELETE "http://localhost:5601/api/apm/sourcemaps/apm:foo-1.0.0-644fd5a9"
 *******************************************************
 ////
 
+[role="xpack"]
+[[android-sourcemap-api]]
+=== Android source map API
+
+IMPORTANT: This endpoint is only compatible with the
+{apm-guide-ref}/index.html[APM integration for Elastic Agent].
+
+An Android source map (generated using Android's https://developer.android.com/build/shrink-code[R8 tool])
+allows obfuscated app stacktraces to be mapped back to original source code --
+allowing you to maintain the size and security of minimized code, without losing the ability to debug your application.
+
+For best results, uploading source maps should become a part of your deployment procedure,
+and not something you only do when you see unhelpful errors.
+That’s because uploading source maps after errors happen won’t make old errors magically readable --
+errors must occur again for source mapping to occur.
+
+The following APIs are available:
+
+* <<android-sourcemap-post>>
+* <<android-sourcemap-get>>
+* <<android-sourcemap-delete>>
+
+[float]
+[[use-android-sourcemap-api]]
+==== How to use APM APIs
+
+.Expand for required headers, privileges, and usage details
+[%collapsible%closed]
+======
+include::api.asciidoc[tag=using-the-APIs]
+======
+
+////
+*******************************************************
+////
+
+[[android-sourcemap-post]]
+==== Create or update an Android source map
+
+Create or update an Android source map for a specific app and version.
+
+[[android-sourcemap-post-privs]]
+===== Privileges
+
+The user accessing this endpoint requires `All` Kibana privileges for the {beat_kib_app} feature.
+For more information, see <<kibana-privileges>>.
+
+[[android-sourcemap-post-req]]
+===== Request
+
+`POST /api/apm/androidmaps`
+
+[role="child_attributes"]
+[[android-sourcemap-post-req-body]]
+===== Request body
+
+`service_name`::
+(required, string) The name of the Android app that the map should apply to.
+
+`service_version`::
+(required, string) The version of the Android app that the map should apply to.
+
+`map_file`::
+(required, string or file upload) The R8-generated map.
+
+[[android-sourcemap-post-example]]
+===== Examples
+
+The following example uploads a source map for a app named `foo` and a service version of `1.0.0`:
+
+[source,curl]
+--------------------------------------------------
+curl -X POST "http://localhost:5601/api/apm/androidmaps" \
+-H 'Content-Type: multipart/form-data' \
+-H 'kbn-xsrf: true' \
+-H 'Authorization: ApiKey ${YOUR_API_KEY}' \
+-F 'service_name="foo"' \
+-F 'service_version="1.0.0"' \
+-F 'map_file=@"/Path/to/the/file/mapping.txt"'
+--------------------------------------------------
+
+[[android-sourcemap-post-body]]
+===== Response body
+
+[source,js]
+--------------------------------------------------
+{
+  "type": "sourcemap",
+  "identifier": "foo-1.0.0-android",
+  "relative_url": "/api/fleet/artifacts/foo-1.0.0-android/644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456",
+  "body": "eJyFkL1OwzAUhd/Fc+MbYMuCEBIbHRjKgBgc96R16tiWr1OQqr47NwqJxEK3q/PzWccXxchnZ7E1A1SjuhjVZtF2yOxiEPlO17oWox3D3uPFeSRTjmJQARfCPeiAgGx8NTKsYdAc1T3rwaSJGcds8Sp3c1HnhfywUZ3QhMTFFGepZxqMC9oex3CS9tpk1XyozgOlmoVKuJX1DqEQZ0su7PGtLU+V/3JPKc3cL7TJ2FNDRPov4bFta3MDM4f7W69lpJjLO9qdK8bzVPhcJz3HUCQ4LbO/p5hCSC4cZPByrp/wFqOklbpefwAhzpqI",
+  "created": "2021-07-09T20:47:44.812Z",
+  "id": "apm:foo-1.0.0-android-644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456",
+  "compressionAlgorithm": "zlib",
+  "decodedSha256": "644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456",
+  "decodedSize": 441,
+  "encodedSha256": "024c72749c3e3dd411b103f7040ae62633558608f480bce4b108cf5b2275bd24",
+  "encodedSize": 237,
+  "encryptionAlgorithm": "none",
+  "packageName": "apm"
+}
+--------------------------------------------------
+
+////
+*******************************************************
+////
+
+[[android-sourcemap-get]]
+==== Get source maps
+
+Returns an array of Fleet artifacts, including source map uploads.
+
+[[android-sourcemap-get-privs]]
+===== Privileges
+
+The user accessing this endpoint requires `Read` or `All` Kibana privileges for the {beat_kib_app} feature.
+For more information, see <<kibana-privileges>>.
+
+[[android-sourcemap-get-req]]
+===== Request
+
+`GET /api/apm/sourcemaps`
+
+[[android-sourcemap-get-example]]
+===== Example
+
+The following example requests all uploaded source maps:
+
+[source,curl]
+--------------------------------------------------
+curl -X GET "http://localhost:5601/api/apm/sourcemaps" \
+-H 'Content-Type: application/json' \
+-H 'kbn-xsrf: true' \
+-H 'Authorization: ApiKey ${YOUR_API_KEY}'
+--------------------------------------------------
+
+[[android-sourcemap-get-body]]
+===== Response body
+
+[source,js]
+--------------------------------------------------
+{
+  "artifacts": [
+    {
+      "type": "sourcemap",
+      "identifier": "foo-1.0.0-android",
+      "relative_url": "/api/fleet/artifacts/foo-1.0.0-android/644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456",
+      "body": {
+        "serviceName": "foo",
+        "serviceVersion": "1.0.0",
+        "bundleFilepath": "android",
+        "sourceMap": "# compiler: R8\n# compiler_version: 3.2.47\n# min_api: 26\n..."
+      },
+      "created": "2021-07-09T20:47:44.812Z",
+      "id": "apm:foo-1.0.0-android-644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456",
+      "compressionAlgorithm": "zlib",
+      "decodedSha256": "644fd5a997d1ddd90ee131ba18e2b3d03931d89dd1fe4599143c0b3264b3e456",
+      "decodedSize": 441,
+      "encodedSha256": "024c72749c3e3dd411b103f7040ae62633558608f480bce4b108cf5b2275bd24",
+      "encodedSize": 237,
+      "encryptionAlgorithm": "none",
+      "packageName": "apm"
+    }
+  ]
+}
+--------------------------------------------------
+
+////
+*******************************************************
+////
+
+[[android-sourcemap-delete]]
+==== Delete source map
+
+Delete a previously uploaded source map.
+
+[[android-sourcemap-delete-privs]]
+===== Privileges
+
+The user accessing this endpoint requires `All` Kibana privileges for the {beat_kib_app} feature.
+For more information, see <<kibana-privileges>>.
+
+[[android-sourcemap-delete-req]]
+===== Request
+
+`DELETE /api/apm/sourcemaps/:id`
+
+[[android-sourcemap-delete-example]]
+===== Example
+
+The following example deletes a source map with an id of `apm:foo-1.0.0-android-644fd5a9`:
+
+[source,curl]
+--------------------------------------------------
+curl -X DELETE "http://localhost:5601/api/apm/sourcemaps/apm:foo-1.0.0-android-644fd5a9" \
+-H 'Content-Type: application/json' \
+-H 'kbn-xsrf: true' \
+-H 'Authorization: ApiKey ${YOUR_API_KEY}'
+--------------------------------------------------
+
+[[android-sourcemap-delete-body]]
+===== Response body
+
+[source,js]
+--------------------------------------------------
+{}
+--------------------------------------------------
+
+////
+*******************************************************
+*******************************************************
+////
+
 [role="xpack"]
 [[agent-key-api]]
 === APM agent Key API

packages/kbn-test/src/kbn_client/kbn_client_requester.ts

@@ -131,6 +131,7 @@ export class KbnClientRequester {
           headers: {
             ...options.headers,
             'kbn-xsrf': 'kbn-client',
+            'x-elastic-internal-origin': 'kbn-client',
           },
           httpsAgent: this.httpsAgent,
           responseType: options.responseType,

x-pack/plugins/apm/server/routes/fleet/source_maps.ts

@@ -19,12 +19,18 @@ import { getPackagePolicyWithSourceMap } from './get_package_policy_decorators';
 
 const doUnzip = promisify(unzip);
 
-interface ApmSourceMapArtifactBody {
+interface ApmMapArtifactBody {
   serviceName: string;
   serviceVersion: string;
   bundleFilepath: string;
+  sourceMap: string;
+}
+
+interface ApmSourceMapArtifactBody
+  extends Omit<ApmMapArtifactBody, 'sourceMap'> {
   sourceMap: SourceMap;
 }
+
 export type ArtifactSourceMap = Omit<Artifact, 'body'> & {
   body: ApmSourceMapArtifactBody;
 };
@@ -104,6 +110,23 @@ export async function createFleetSourceMapArtifact({
   });
 }
 
+export async function createFleetAndroidMapArtifact({
+  apmArtifactBody,
+  fleetPluginStart,
+}: {
+  apmArtifactBody: ApmMapArtifactBody;
+  fleetPluginStart: FleetPluginStart;
+}) {
+  const apmArtifactClient = getApmArtifactClient(fleetPluginStart);
+  const identifier = `${apmArtifactBody.serviceName}-${apmArtifactBody.serviceVersion}-android`;
+
+  return apmArtifactClient.createArtifact({
+    type: 'sourcemap',
+    identifier,
+    content: JSON.stringify(apmArtifactBody),
+  });
+}
+
 export async function deleteFleetSourcemapArtifact({
   id,
   fleetPluginStart,

x-pack/plugins/apm/server/routes/source_maps/bulk_create_apm_source_maps.ts

@@ -10,7 +10,7 @@ import { Artifact } from '@kbn/fleet-plugin/server';
 import { getUnzippedArtifactBody } from '../fleet/source_maps';
 import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices';
 import { ApmSourceMap } from './create_apm_source_map_index_template';
-import { getEncodedContent, getSourceMapId } from './sourcemap_utils';
+import { getEncodedSourceMapContent, getSourceMapId } from './sourcemap_utils';
 
 export async function bulkCreateApmSourceMaps({
   artifacts,
@@ -24,7 +24,7 @@ export async function bulkCreateApmSourceMaps({
       const { serviceName, serviceVersion, bundleFilepath, sourceMap } =
         await getUnzippedArtifactBody(artifact.body);
 
-      const { contentEncoded, contentHash } = await getEncodedContent(
+      const { contentEncoded, contentHash } = await getEncodedSourceMapContent(
         sourceMap
       );
 

x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map.ts

@@ -10,7 +10,11 @@ import { Logger } from '@kbn/core/server';
 import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices';
 import { ApmSourceMap } from './create_apm_source_map_index_template';
 import { SourceMap } from './route';
-import { getEncodedContent, getSourceMapId } from './sourcemap_utils';
+import {
+  getEncodedSourceMapContent,
+  getEncodedContent,
+  getSourceMapId,
+} from './sourcemap_utils';
 
 export async function createApmSourceMap({
   internalESClient,
@@ -31,9 +35,75 @@ export async function createApmSourceMap({
   serviceName: string;
   serviceVersion: string;
 }) {
-  const { contentEncoded, contentHash } = await getEncodedContent(
+  const { contentEncoded, contentHash } = await getEncodedSourceMapContent(
     sourceMapContent
   );
+  return await doCreateApmMap({
+    internalESClient,
+    logger,
+    fleetId,
+    created,
+    bundleFilepath,
+    serviceName,
+    serviceVersion,
+    contentEncoded,
+    contentHash,
+  });
+}
+
+export async function createApmAndroidMap({
+  internalESClient,
+  logger,
+  fleetId,
+  created,
+  mapContent,
+  bundleFilepath,
+  serviceName,
+  serviceVersion,
+}: {
+  internalESClient: ElasticsearchClient;
+  logger: Logger;
+  fleetId: string;
+  created: string;
+  mapContent: string;
+  bundleFilepath: string;
+  serviceName: string;
+  serviceVersion: string;
+}) {
+  const { contentEncoded, contentHash } = await getEncodedContent(mapContent);
+  return await doCreateApmMap({
+    internalESClient,
+    logger,
+    fleetId,
+    created,
+    bundleFilepath,
+    serviceName,
+    serviceVersion,
+    contentEncoded,
+    contentHash,
+  });
+}
+async function doCreateApmMap({
+  internalESClient,
+  logger,
+  fleetId,
+  created,
+  bundleFilepath,
+  serviceName,
+  serviceVersion,
+  contentEncoded,
+  contentHash,
+}: {
+  internalESClient: ElasticsearchClient;
+  logger: Logger;
+  fleetId: string;
+  created: string;
+  bundleFilepath: string;
+  serviceName: string;
+  serviceVersion: string;
+  contentEncoded: string;
+  contentHash: string;
+}) {
   const doc: ApmSourceMap = {
     fleet_id: fleetId,
     created,