[ML] ESS Kibana API integration tests fail with Error: self-signed certificate in certificate chain when calling getService('ml').importTrainedModel() with ssl:true configured

[spong]

Build Details

Kibana version: main/8.16 3bea483
Elasticsearch version: Same, run via Kibana API Integration Server

Summary

In ESS Kibana API integration tests, when calling getService('ml').importTrainedModel() with ssl:true configured, the test fails with the following Error: self-signed certificate in certificate chain:

     └-> "before all" hook: beforeTestSuite.trigger in "@ess Basic Security AI Assistant Knowledge Base Entries"
     └-> "before all" hook in "@ess Basic Security AI Assistant Knowledge Base Entries"
       │ debg Waiting up to 60000ms for Stats index to exist...
       │ debg Creating trained model with id "pt_tiny_elser"
       │ debg > Trained model created
       │ debg Creating vocabulary for trained model "pt_tiny_elser"
       └- ✖ fail: GenAI - Knowledge Base Entries APIs @ess Basic Security AI Assistant Knowledge Base Entries "before all" hook in "@ess Basic Security AI Assistant Knowledge Base Entries"
       │      Error: self-signed certificate in certificate chain
       │       at TLSSocket.onConnectSecure (node:_tls_wrap:1674:34)
       │       at TLSSocket.emit (node:events:519:28)
       │       at TLSSocket._finishInit (node:_tls_wrap:1085:8)
       │       at TLSWrap.ssl.onhandshakedone (node:_tls_wrap:871:12)

From the │ debg > Trained model created log we can see the createTrainedModel call to /internal/ml/trained_models/${modelId} was successful:

kibana/x-pack/test/functional/services/ml/api.ts

Lines 1381 to 1391 in f18224c

	async createTrainedModel(modelId: string, body: PutTrainedModelConfig, space?: string) {
	log.debug(`Creating trained model with id "${modelId}"`);
	const { body: model, status } = await kbnSupertest
	.put(`${space ? `/s/${space}` : ''}/internal/ml/trained_models/${modelId}`)
	.set(getCommonRequestHeader('1'))
	.send(body);
	this.assertResponseStatusCode(200, status, model);
	log.debug('> Trained model created');
	return model;
	},

However the subsequent createTrainedModelVocabularyES call to /_ml/trained_models/${modelId}/vocabulary, surfaces the error:

kibana/x-pack/test/functional/services/ml/api.ts

Lines 1404 to 1412 in f18224c

	async createTrainedModelVocabularyES(modelId: string, body: TrainedModelVocabulary) {
	log.debug(`Creating vocabulary for trained model "${modelId}"`);
	const { body: responseBody, status } = await esSupertest
	.put(`/_ml/trained_models/${modelId}/vocabulary`)
	.send(body);
	this.assertResponseStatusCode(200, status, responseBody);
	log.debug('> Trained model vocabulary created');
	},

Steps to reproduce

Create an API Integration test that calls getService('ml').importTrainedModel(). This is the configuration I used, which was within the security_solution_api_integration directory, extending from the config.base.trial config:

ess.config.ts

import { FtrConfigProviderContext } from '@kbn/test';

export default async function ({ readConfigFile }: FtrConfigProviderContext) {
  const functionalConfig = await readConfigFile(
    require.resolve('../../../../../../config/ess/config.base.trial')
  );

  return {
    ...functionalConfig.getAll(),
    kbnTestServer: {
      ...functionalConfig.get('kbnTestServer'),
      serverArgs: [
        ...functionalConfig.get('kbnTestServer.serverArgs'),
        `--xpack.securitySolution.enableExperimental=${JSON.stringify([
          'assistantKnowledgeBaseByDefault',
        ])}`,
      ],
    },
    testFiles: [require.resolve('..')],
    junit: {
      reportName: 'GenAI - Knowledge Base Entries Tests - ESS Env - Trial License',
    },
  };
}

ess.config.json

{
  "testFiles": [
    "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/index.ts"
  ],
  "servers": {
    "kibana": {
      "protocol": "http",
      "hostname": "localhost",
      "port": 5620,
      "auth": "elastic:changeme",
      "username": "elastic",
      "password": "changeme"
    },
    "elasticsearch": {
      "protocol": "https",
      "hostname": "localhost",
      "port": 9220,
      "auth": "system_indices_superuser:changeme",
      "username": "system_indices_superuser",
      "password": "changeme"
    },
    "fleetserver": {
      "protocol": "http",
      "hostname": "localhost"
    }
  },
  "services": {},
  "junit": {
    "reportName": "GenAI - Knowledge Base Entries Tests - ESS Env - Trial License"
  },
  "esTestCluster": {
    "license": "trial",
    "from": "snapshot",
    "serverArgs": [
      "xpack.license.self_generated.type=trial"
    ],
    "ssl": true
  },
  "kbnTestServer": {
    "buildArgs": [],
    "sourceArgs": [
      "--no-base-path",
      "--env.name=development"
    ],
    "serverArgs": [
      "--server.port=5620",
      "--status.allowAnonymous=true",
      "--elasticsearch.hosts=http://localhost:9220",
      "--elasticsearch.username=kibana_system",
      "--elasticsearch.password=changeme",
      "--data.search.aggs.shardDelay.enabled=true",
      "--data.query.timefilter.minRefreshInterval=1000",
      "--security.showInsecureClusterWarning=false",
      "--telemetry.banner=false",
      "--telemetry.optIn=false",
      "--telemetry.sendUsageTo=staging",
      "--server.maxPayload=1679958",
      "--plugin-path=/Users/garrettspong/dev/kibana-main/test/common/plugins/newsfeed",
      "--plugin-path=/Users/garrettspong/dev/kibana-main/test/common/plugins/otel_metrics",
      "--newsfeed.service.urlRoot=http://localhost:5620",
      "--newsfeed.service.pathTemplate=/api/_newsfeed-FTS-external-service-simulators/kibana/v{VERSION}.json",
      "--logging.appenders.deprecation={\"type\":\"console\",\"layout\":{\"type\":\"json\"}}",
      "--logging.loggers=[{\"name\":\"elasticsearch.deprecation\",\"level\":\"all\",\"appenders\":[\"deprecation\"]}]",
      "--logging.appenders.default={\"type\":\"console\",\"layout\":{\"type\":\"pattern\",\"pattern\":\"[%date][%level][%logger] %message %meta\"}}",
      "--logging.appenders.console={\"type\":\"console\",\"layout\":{\"type\":\"pattern\",\"pattern\":\"[%date][%level][%logger] %message %meta\"}}",
      "--status.allowAnonymous=true",
      "--server.uuid=5b2de169-2785-441b-ae8c-186a1936b17d",
      "--xpack.maps.showMapsInspectorAdapter=true",
      "--xpack.maps.preserveDrawingBuffer=true",
      "--xpack.security.encryptionKey=\"wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf\"",
      "--xpack.encryptedSavedObjects.encryptionKey=\"DkdXazszSCYexXqz4YktBGHCRkV6hyNK\"",
      "--xpack.discoverEnhanced.actions.exploreDataInContextMenu.enabled=true",
      "--savedObjects.maxImportPayloadBytes=10485760",
      "--savedObjects.allowHttpApiAccess=false",
      "--server.restrictInternalApis=false",
      "--xpack.task_manager.unsafe.exclude_task_types=[\"Fleet-Metrics-Task\"]",
      "--xpack.security.session.idleTimeout=3600000",
      "--telemetry.optIn=true",
      "--xpack.fleet.agents.pollingRequestTimeout=5000",
      "--xpack.ruleRegistry.write.enabled=true",
      "--xpack.ruleRegistry.write.enabled=true",
      "--xpack.ruleRegistry.write.cache.enabled=false",
      "--monitoring_collection.opentelemetry.metrics.prometheus.enabled=true",
      "--xpack.actions.allowedHosts=[\"localhost\",\"some.non.existent.com\"]",
      "--xpack.actions.enabledActionTypes=[\".cases\",\".email\",\".index\",\".pagerduty\",\".swimlane\",\".server-log\",\".servicenow\",\".slack\",\".webhook\",\"test.authorization\",\"test.failing\",\"test.index-record\",\"test.noop\",\"test.rate-limit\"]",
      "--xpack.eventLog.logEntries=true",
      "--xpack.securitySolution.alertIgnoreFields=[\"testing_ignored.constant\",\"/testing_regex*/\"]",
      "--xpack.ruleRegistry.write.enabled=true",
      "--xpack.ruleRegistry.write.cache.enabled=false",
      "--xpack.ruleRegistry.unsafe.indexUpgrade.enabled=true",
      "--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true",
      "--xpack.securitySolution.enableExperimental=[\"previewTelemetryUrlEnabled\",\"riskScoringPersistence\",\"riskScoringRoutesEnabled\",\"manualRuleRunEnabled\"]",
      "--xpack.task_manager.poll_interval=1000",
      "--xpack.actions.preconfigured={\"my-test-email\":{\"actionTypeId\":\".email\",\"name\":\"TestEmail#xyz\",\"config\":{\"from\":\"me@test.com\",\"service\":\"__json\"},\"secrets\":{\"user\":\"user\",\"password\":\"password\"}}}",
      "--elasticsearch.hosts=https://localhost:9220",
      "--elasticsearch.ssl.certificateAuthorities=/Users/garrettspong/dev/kibana-main/packages/kbn-dev-utils/certs/ca.crt",
      "--xpack.securitySolution.enableExperimental=[\"assistantKnowledgeBaseByDefault\"]"
    ],
    "useDedicatedTaskRunner": false,
    "runOptions": {
      "wait": {},
      "alwaysUseSource": false
    },
    "env": {
      "ELASTICSEARCH_USERNAME": "elastic"
    }
  },
  "mochaOpts": {
    "grep": "/^(?!.*@skipInEss).*@ess.*/",
    "bail": false,
    "dryRun": false,
    "invert": false,
    "slow": 30000,
    "timeout": 360000,
    "ui": "bdd"
  },
  "serverless": false,
  "suiteFiles": {
    "include": [],
    "exclude": []
  },
  "suiteTags": {
    "include": [],
    "exclude": []
  },
  "servicesRequiredForTestAnalysis": [],
  "pageObjects": {},
  "timeouts": {
    "find": 10000,
    "try": 120000,
    "waitFor": 20000,
    "esRequestTimeout": 30000,
    "kibanaReportCompletion": 60000,
    "kibanaStabilize": 15000,
    "navigateStatusPageCheck": 250,
    "waitForExists": 2500
  },
  "updateBaselines": false,
  "updateSnapshots": false,
  "browser": {
    "type": "chrome",
    "logPollingMs": 100,
    "acceptInsecureCerts": false
  },
  "mochaReporter": {
    "captureLogOutput": false,
    "sendToCiStats": false
  },
  "esServerlessOptions": {
    "resources": []
  },
  "chromedriver": {
    "url": "http://localhost:9515"
  },
  "firefoxdriver": {
    "url": "http://localhost:2828"
  },
  "apps": {},
  "esArchiver": {},
  "kbnArchiver": {
    "directory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/fixtures/kbn_archiver"
  },
  "uiSettings": {},
  "screenshots": {
    "directory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/screenshots"
  },
  "snapshots": {
    "directory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/snapshots"
  },
  "failureDebugging": {
    "htmlDirectory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/failure_debug/html"
  },
  "layout": {
    "fixedHeaderHeight": 100
  },
  "security": {
    "roles": {},
    "defaultRoles": [
      "superuser"
    ]
  },
  "dockerServers": {}
}

integration_test.ts

export default ({ getService }: FtrProviderContext) => {
  const supertest = getService('supertest');
  const ml = getService('ml') as ReturnType<typeof MachineLearningProvider>;

  describe('@ess Repro self-signed certificate error', () => {
    before(async () => {
      const config = {
        ...ml.api.getTrainedModelConfig(TINY_ELSER.name),
        input: {
          field_names: ['text_field'],
        },
      };
      await ml.api.assureMlStatsIndexExists();
      await ml.api.importTrainedModel(TINY_ELSER.name, TINY_ELSER.id, config);
    });

    after(async () => {
      await ml.api.stopTrainedModelDeploymentES(TINY_ELSER.id, true);
      await ml.api.deleteTrainedModelES(TINY_ELSER.id);
      await ml.api.cleanMlIndices();
      await ml.testResources.cleanMLSavedObjects();
    });

    describe('Create Entries', () => {
      it('should say hi', async () => {
        console.log('hi');
      });
    });
  });
};

Workaround

Disabling SSL via ssl:false in the configuration prevents this issue from occurring, so since the other parts of my tests do not require SSL I've disabled it in my suite for the time being.

[elasticmachine]

Pinging @elastic/ml-ui (:ml)