[Fleet] Identify packages with missing references

[joshdover]

In #127664 it was discovered that some packages contain Saved Objects with missing references to other Saved Objects. In this particular case, we were able to remove the missing reference, but we also updated the package install code to handle objects with missing references by installing them anyways.

In order to make package installation more robust, we should spend some time investigating how many packages contain missing references and determine if this is something we should continue to support or if we should start validating this as part of the package-spec and prevent installations of these packages in the future.

@hop-dev has a neat script for installing all the packages, copied from #127760 (comment):

install_all_packages.js

import fetch from "node-fetch";

const SKIP_PACKAGES = ["kibana", "logstash", "microsoft", "fleet_server", "elastic_agent"]
async function installPackage(name, version) {
    const start = Date.now()
    const res = await fetch(`http://localhost:5601/mark/api/fleet/epm/packages/${name}-${version}`, {
        "headers": {
            "accept": "*/*",
            "content-type": "application/json",
            "kbn-xsrf": "xyz",
            'Authorization': 'Basic ' + Buffer.from("elastic:changeme").toString('base64')
        },
        "body": null,
        "method": "POST"
    });
    const end = Date.now()

    const body = await res.json()

    return { body, status: res.status, took : (end - start) / 1000 };
}

async function deletePackage(name, version) {
    const res = await fetch(`http://localhost:5601/mark/api/fleet/epm/packages/${name}-${version}`, {
        "headers": {
            "accept": "*/*",
            "content-type": "application/json",
            "kbn-xsrf": "xyz",
            'Authorization': 'Basic ' + Buffer.from("elastic:changeme").toString('base64')
        },
        "body": null,
        "method": "DELETE"
    });

    const body = await res.json()

    return { body, status: res.status };
}

async function getAllPackages() {
const res  = await fetch("https://epr-snapshot.elastic.co/search", {
  "headers": {
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
  },
  "body": null,
  "method": "GET"
});
const body = await res.json()
    return body;
}

function logResult(pkg, result){
    const pre = `${pkg.name}-${pkg.version} ${result.took ? ` took ${result.took}s` : ""} : `;
    if (result.status !== 200) {
        console.log(pre + JSON.stringify(result))
    } else {
        console.log(pre + 200)
    }
}

async function go() {
    const allPackages = await getAllPackages()

    console.log("INSTALLING ------");

    for (const pkg of allPackages) {
        if (SKIP_PACKAGES.includes(pkg.name)) {
            console.log(`Skipping ${pkg.name}`)
            continue;
        }
        const result = await installPackage(pkg.name, pkg.version);

        logResult(pkg,result)
    }

    const deletePackages = process.argv.includes("--delete")

    if(!deletePackages) return;
    
    console.log("DELETING ------");
    for (const pkg of allPackages) {
        if (SKIP_PACKAGES.includes(pkg.name)) {
            console.log(`Skipping ${pkg.name}`)
            continue;
        }
        const result = await deletePackage(pkg.name, pkg.version);

        logResult(pkg,result)
    }
}

go()

[elasticmachine]

Pinging @elastic/fleet (Team:Fleet)

[nchaulet]

I just runned the @hop-dev tools on a branch that throw on reference error and it seems aws was the only one with issue

the window package seems to have a mapping error and an analyzer I am wondering if we broke that when moving the mapping. (I created that issue for this problem #128209)

INSTALLING ------


✅ 1password-1.2.0  took 9.869s : 200
✅ aws-1.13.1  took 10.252s : 200
✅ ti_abusech-1.2.0  took 10.323s : 200
✅ activemq-0.3.0  took 10.304s : 200
✅ akamai-0.1.1  took 8.168s : 200
✅ ti_otx-1.2.0  took 10.411s : 200
✅ ti_anomali-1.2.1  took 10.393s : 200
✅ apache-1.3.5  took 10.254s : 200
✅ tomcat-1.3.0  took 8.164s : 200
✅ netscout-0.7.0  took 8.229s : 200
✅ atlassian_bitbucket-1.1.0  took 8.718s : 200
✅ atlassian_confluence-1.1.0  took 8.231s : 200
✅ atlassian_jira-1.1.0  took 8.208s : 200
✅ auditd-2.1.0  took 10.292s : 200
✅ auth0-0.1.2  took 10.301s : 200
✅ azure_application_insights-1.0.1  took 10.339s : 200
✅ azure_billing-1.0.1  took 10.293s : 200
✅ azure-1.1.1  took 10.252s : 200
✅ azure_metrics-1.0.2  took 10.335s : 200
✅ barracuda-0.8.0  took 8.26s : 200
✅ bluecoat-0.7.0  took 8.153s : 200
✅ cef-1.4.2  took 10.214s : 200
✅ cassandra-1.2.0  took 10.261s : 200
✅ checkpoint-1.3.1  took 8.252s : 200
✅ cisco_asa-2.2.0  took 10.287s : 200
✅ cisco_duo-1.1.0  took 10.245s : 200
✅ cisco_ftd-2.0.0  took 8.155s : 200
✅ cisco_ios-1.4.1  took 8.175s : 200
✅ cisco_meraki-0.4.0  took 8.273s : 200
✅ cisco_nexus-0.4.0  took 8.174s : 200
✅ cisco_secure_endpoint-2.0.0  took 8.177s : 200
✅ cisco_umbrella-0.5.0  took 8.176s : 200
✅ cloudflare-1.3.1  took 10.221s : 200
✅ cockroachdb-0.2.1  took 10.207s : 200
✅ containerd-0.1.0  took 10.177s : 200
✅ crowdstrike-1.2.3  took 14.222s : 200
✅ aws_logs-0.2.1  took 7.139s : 200
✅ gcp_pubsub-1.0.0  took 8.127s : 200
✅ http_endpoint-1.0.1  took 7.163s : 200
✅ httpjson-1.1.0  took 7.2s : 200
✅ log-1.0.0  took 7.15s : 200
✅ tcp-1.0.0  took 7.132s : 200
✅ udp-1.0.1  took 7.12s : 200
✅ winlog-1.4.0  took 7.132s : 200
✅ cyberarkpas-2.3.0  took 9.168s : 200
✅ ti_cybersixgill-1.3.0  took 10.164s : 200
✅ cylance-0.7.0  took 10.229s : 200
✅ docker-2.0.0  took 21.481s : 200
✅ apm-8.2.0-dev4  took 24.327s : 200
✅ elastic_agent-1.3.1  took 10.238s : 200
✅ synthetics-0.9.2  took 13.15s : 200
✅ endpoint-1.5.0  took 32.378s : 200
✅ f5-0.8.0  took 10.219s : 200
✅ fireeye-1.2.0  took 10.22s : 200
✅ fleet_server-1.1.1  took 4.105s : 200
✅ fortinet-1.4.2  took 13.277s : 200
✅ github-0.3.2  took 9.163s : 200
✅ gcp-1.4.2  took 15.296s : 200
✅ santa-2.0.0  took 10.32s : 200
✅ google_workspace-1.3.1  took 16.18s : 200
✅ haproxy-1.1.0  took 11.247s : 200
✅ hashicorp_vault-1.3.1  took 14.355s : 200
✅ hid_bravura_monitor-1.0.1  took 11.204s : 200
✅ iis-0.8.2  took 14.208s : 200
✅ imperva-0.7.0  took 9.155s : 200
✅ infoblox-0.7.0  took 9.164s : 200
✅ iptables-0.8.0  took 10.422s : 200
✅ juniper_junos-0.1.0  took 9.204s : 200
✅ juniper-1.1.0  took 15.503s : 200
✅ juniper_netscreen-0.1.0  took 9.172s : 200
✅ juniper_srx-1.1.1  took 11.17s : 200
✅ kafka-1.2.2  took 14.186s : 200
✅ keycloak-1.2.0  took 9.142s : 200
✅ kubernetes-1.17.3  took 44.285s : 200
✅ linux-0.6.2  took 22.184s : 200
✅ logstash-1.1.0  took 15.207s : 200
✅ ti_misp-1.2.0  took 10.169s : 200
✅ mattermost-1.1.0  took 10.159s : 200
✅ microsoft_dhcp-1.3.0  took 10.181s : 200
✅ microsoft_defender_endpoint-2.0.0  took 10.248s : 200
✅ microsoft_sqlserver-0.4.2  took 10.188s : 200
✅ mimecast-0.0.7  took 20.218s : 200
✅ modsecurity-0.1.4  took 9.164s : 200
✅ mongodb-1.3.1  took 16.23s : 200
✅ mysql-1.3.0  took 15.199s : 200
✅ mysql_enterprise-1.0.0  took 9.176s : 200
✅ nats-1.3.0  took 20.192s : 200
✅ netflow-1.4.0  took 10.194s : 200
✅ netskope-0.1.0  took 11.214s : 200
✅ network_traffic-0.7.1  took 38.28s : 200
✅ nginx-1.3.1  took 15.243s : 200
✅ nginx_ingress_controller-1.3.0  took 12.215s : 200
✅ o365-1.4.1  took 11.202s : 200
✅ okta-1.5.0  took 10.18s : 200
✅ oracle-1.0.0  took 9.201s : 200
✅ osquery-1.2.0  took 10.18s : 200
✅ osquery_manager-1.0.0  took 8.129s : 200
✅ panw_cortex_xdr-1.1.0  took 9.156s : 200
✅ panw-1.5.1  took 12.206s : 200
✅ postgresql-1.2.0  took 17.245s : 200
✅ security_detection_engine-1.0.1  took 6.13s : 200
✅ prometheus-0.9.0  took 13.203s : 200
✅ proofpoint-0.6.0  took 10.131s : 200
✅ pulse_connect_secure-0.2.0  took 10.192s : 200
✅ qnap_nas-1.1.0  took 10.297s : 200
✅ rabbitmq-1.3.0  took 16.236s : 200
✅ radware-0.6.0  took 10.165s : 200
✅ ti_recordedfuture-0.1.0  took 9.166s : 200
✅ redis-1.3.0  took 18.19s : 200
✅ stan-1.3.0  took 14.277s : 200
✅ snapshot-0.0.1  took 4.149s : 200
✅ snort-0.2.0  took 9.158s : 200
✅ snyk-1.1.0  took 11.178s : 200
✅ sonicwall-0.7.0  took 9.165s : 200
✅ sophos-1.2.1  took 16.173s : 200
✅ squid-0.7.0  took 9.25s : 200
✅ staging-0.0.1  took 4.067s : 200
✅ suricata-1.6.0  took 12.165s : 200
✅ symantec_endpoint-0.0.1  took 10.186s : 200
✅ system-1.11.0  took 10.243s : 200
✅ tenable_sc-1.1.0  took 15.242s : 200
✅ ti_threatq-1.2.0  took 11.227s : 200
✅ traefik-1.3.0  took 13.171s : 200
✅ carbonblack_edr-1.1.0  took 10.164s : 200
✅ vsphere-0.1.0  took 16.254s : 200
❌ windows-1.10.0  took 22.412s : {"body":{"statusCode":500,"error":"Internal Server Error","message":"mapper_parsing_exception: [mapper_parsing_exception] Reason: Failed to parse mapping: analyzer [powershell_script_analyzer] has not been configured in mappings"},"status":500,"took":22.412}
✅ zeek-1.6.0  took 110.469s : 200
✅ zerofox-1.2.0  took 9.227s : 200
✅ zookeeper-1.3.0  took 12.296s : 200
✅ zoom-1.2.0  took 14.23s : 200
✅ zscaler_zia-0.1.1  took 21.094s : 200
✅ zscaler-0.1.2  took 9.189s : 200
✅ zscaler_zpa-0.1.1  took 22.507s : 200
✅ etcd-0.1.1  took 17.199s : 200
✅ pfsense-0.3.0  took 14.199s : 200

[joshdover]

This feels like a really useful test for us to have (ensuring all packages can be installed by Kibana), at least at the e2e level.

[joshdover]

Let's track the remaining work in #128493