diff --git a/.changeset/selfish-beds-brake.md b/.changeset/selfish-beds-brake.md new file mode 100644 index 00000000..53aeef51 --- /dev/null +++ b/.changeset/selfish-beds-brake.md @@ -0,0 +1,5 @@ +--- +'@hashicorp/integrations-hcl': minor +--- + +Support a 'strategy' field for multiple integrations-hcl consumption paths. Includes a strategy for nomad-pack. diff --git a/packages/integrations-hcl/index.ts b/packages/integrations-hcl/index.ts index f0541759..75c15041 100644 --- a/packages/integrations-hcl/index.ts +++ b/packages/integrations-hcl/index.ts @@ -2,76 +2,71 @@ import * as fs from 'fs' import { glob } from 'glob' import * as path from 'path' import { z } from 'zod' -import { IntegrationsAPI, VariableGroupConfig } from './lib/generated' -import HCL from './lib/hcl' -import { - Component, - Integration, - Variable, - VariableGroup, -} from './schemas/integration' -import MetadataHCLSchema from './schemas/metadata.hcl' -import { getVariablesSchema } from './schemas/variables.hcl' +import { IntegrationsAPI } from './lib/generated' +import { Integration } from './schemas/integration' +import { loadDefaultIntegrationDirectory } from './strategies/default/load_default_directory' +import { loadNomadPackIntegrationDirectory } from './strategies/nomad-pack/load_pack_directory' const Config = z.object({ identifier: z.string(), repo_path: z.string(), version: z.string(), + strategy: z.enum(['default', 'nomad-pack']).default('default').optional(), }) type Config = z.infer export default async function LoadFilesystemIntegration( config: Config ): Promise { - // Create the API client - const client = new IntegrationsAPI({ + // Create an API client instance + const apiClient = new IntegrationsAPI({ BASE: process.env.INPUT_INTEGRATIONS_API_BASE_URL, }) - // Fetch the Integration from the API that we're looking to update + // Parse & Validate the Integration Identifier const [productSlug, organizationSlug, integrationSlug] = config.identifier.split('/') - - // Throw if the identifier is invalid if (!productSlug || !organizationSlug || !integrationSlug) { + // Throw if the identifier is invalid throw new Error( `Invalid integration identifier: '${config.identifier}'.` + ` The expected format is 'productSlug/organizationSlug/integrationSlug'` ) } - const organization = await client.organizations.fetchOrganization( + // Validate the Organization as specified in the identifier exists + const organization = await apiClient.organizations.fetchOrganization( organizationSlug ) - if (organization.meta.status_code != 200) { throw new Error( `Organization not found for integration identifier: '${config.identifier}'` ) } - const integrationFetchResult = await client.integrations.fetchIntegration( + // Fetch the Integration from the API. We need to ensure that it exists, + // and therefore has already been registered. + const integrationFetchResult = await apiClient.integrations.fetchIntegration( productSlug, organization.result.id, integrationSlug ) - if (integrationFetchResult.meta.status_code !== 200) { throw new Error( `Integration not found for integration identifier: '${config.identifier}'` ) } - const apiIntegration = integrationFetchResult.result - // Parse out & validate the metadata.hcl file - const repoRootDirectory = path.join( + // Determine the location of the Integration & validate that it has a metadata.hcl file + const integrationDirectory = path.join( config.repo_path, apiIntegration.subdirectory || '' ) - const metadataFilePath = path.join(repoRootDirectory, 'metadata.hcl') + const metadataFilePath = path.join(integrationDirectory, 'metadata.hcl') - // Throw if the metadata.hcl file doesn't exist + // Throw if the metadata.hcl file doesn't exist. We don't validate it at + // this point beyond checking that it exists. if (!fs.existsSync(metadataFilePath)) { const matches = await glob('**/metadata.hcl', { cwd: config.repo_path }) // If no metadata.hcl files were found, throw a helpful error @@ -100,152 +95,39 @@ export default async function LoadFilesystemIntegration( } } - // @todo(kevinwang): - // Maybe lift file content reading into HCL class and throw a more helpful error message - const fileContent = fs.readFileSync(metadataFilePath, 'utf8') - const hclConfig = new HCL(fileContent, MetadataHCLSchema) - // throw a verbose error message with the filepath and contents - if (!hclConfig.result.data) { - throw new Error( - hclConfig.result.error.message + - '\n' + - 'File: ' + - metadataFilePath + - '\n' + - fileContent - ) - } - - const hclIntegration = hclConfig.result.data.integration[0] - - // Read the README - let readmeContent: string | null = null - if (hclIntegration.docs[0].process_docs) { - const readmeFile = path.join( - repoRootDirectory, - hclIntegration.docs[0].readme_location - ) - - // Throw if the README file doesn't exist - if (!fs.existsSync(readmeFile)) { - throw new Error( - `The README file, ${readmeFile}, was derived from config values and integration data, but it does not exist.` + - ` ` + - `Please double check the "readme_location" value in ${metadataFilePath}, and try again.` - ) - } - readmeContent = fs.readFileSync(readmeFile, 'utf8') - } - - // Load the Products VariableGroupConfigs so we can load any component variables + // Load the Integration's product VariableGroupConfigs. This is information + // that we need to parse out the Integration from the Filesystem. const variableGroupConfigs = - await client.variableGroupConfigs.fetchVariableGroupConfigs( + await apiClient.variableGroupConfigs.fetchVariableGroupConfigs( apiIntegration.product.slug, '100' ) - if (variableGroupConfigs.meta.status_code !== 200) { throw new Error( `Failed to load 'variable_group' configs for product: '${apiIntegration.product.slug}'` ) } - // Calculate each Component object - const allComponents: Array = [] - for (let i = 0; i < hclIntegration.component.length; i++) { - allComponents.push( - await loadComponent( - repoRootDirectory, - hclIntegration.component[i].type, - hclIntegration.component[i].name, - hclIntegration.component[i].slug, - variableGroupConfigs.result + // Depending on the Strategy that is specified, we read the filesystem and coerce the + // configuration to a standardized Integrations object. + switch (config.strategy) { + case 'nomad-pack': { + return loadNomadPackIntegrationDirectory( + integrationDirectory, + apiIntegration.id, + apiIntegration.product.slug, + config.version ) - ) - } - - // Return Integration with all defaults set - return { - id: apiIntegration.id, - product: apiIntegration.product.slug, - identifier: hclIntegration.identifier, - name: hclIntegration.name, - description: hclIntegration.description, - current_release: { - version: config.version, - readme: readmeContent, - components: allComponents, - }, - flags: hclIntegration.flags, - docs: hclIntegration.docs[0], - hide_versions: hclIntegration.hide_versions, - license: hclIntegration.license[0], - integration_type: hclIntegration.integration_type, - } -} - -async function loadComponent( - repoRootDirectory: string, - componentType: string, - componentName: string, - componentSlug: string, - variableGroupConfigs: Array -): Promise { - // Calculate the location of the folder where the README / variables, etc reside - const componentFolder = `${repoRootDirectory}/components/${componentType}/${componentSlug}` - - // Load the README if it exists - const componentReadmeFile = `${componentFolder}/README.md` - let readmeContent: string | null = null - try { - readmeContent = fs.readFileSync(componentReadmeFile, 'utf8') - } catch (err) { - // No issue, there's just no README, which is OK! - } - - // Go through each VariableGroupConfig to try see if we need to load them - const variableGroups: Array = [] + } - for (let i = 0; i < variableGroupConfigs.length; i++) { - const variableGroupConfig = variableGroupConfigs[i] - const variableGroupFile = `${componentFolder}/${variableGroupConfig.filename}` - if (fs.existsSync(variableGroupFile)) { - // Load & Validate the Variable Files (parameters.hcl, outputs.hcl, etc.) - const fileContent = fs.readFileSync(variableGroupFile, 'utf8') - const hclConfig = new HCL( - fileContent, - getVariablesSchema(variableGroupConfig.stanza) + default: { + return loadDefaultIntegrationDirectory( + integrationDirectory, + apiIntegration.id, + apiIntegration.product.slug, + config.version, + variableGroupConfigs.result ) - if (!hclConfig.result.data) { - throw new Error(hclConfig.result.error.message) - } - - // Map the HCL File variable configuration to the Variable defaults - const variables: Array = hclConfig.result.data[ - variableGroupConfig.stanza - ].map((v) => { - return { - key: v.key, - description: v.description ? v.description : null, - type: v.type ? v.type : null, - required: typeof v.required != 'undefined' ? v.required : null, - default_value: v.default_value ? v.default_value : null, - } - }) - variableGroups.push({ - variable_group_config_id: variableGroupConfig.id, - variables, - }) - } else { - console.warn(`Variable Group File '${variableGroupFile}' not found.`) } } - - return { - type: componentType, - name: componentName, - slug: componentSlug, - readme: readmeContent, - variable_groups: variableGroups, - } } diff --git a/packages/integrations-hcl/strategies/default/load_default_directory.ts b/packages/integrations-hcl/strategies/default/load_default_directory.ts new file mode 100644 index 00000000..f7a34220 --- /dev/null +++ b/packages/integrations-hcl/strategies/default/load_default_directory.ts @@ -0,0 +1,156 @@ +import * as fs from 'fs' +import * as path from 'path' +import { VariableGroupConfig } from '../../lib/generated' +import HCL from '../../lib/hcl' +import { + Component, + Integration, + Variable, + VariableGroup, +} from '../../schemas/integration' +import MetadataHCLSchema from './metadata.hcl' +import { getVariablesSchema } from './variables.hcl' + +export async function loadDefaultIntegrationDirectory( + integrationDirectory: string, + integrationID: string, + integrationProductSlug: string, + currentReleaseVersion: string, + variableGroupConfigs: VariableGroupConfig[] +): Promise { + const metadataFilePath = path.join(integrationDirectory, 'metadata.hcl') + + // Read & Validate the Metadata file + const fileContent = fs.readFileSync(metadataFilePath, 'utf8') + const hclConfig = new HCL(fileContent, MetadataHCLSchema) + // throw a verbose error message with the filepath and contents + if (!hclConfig.result.data) { + throw new Error( + hclConfig.result.error.message + + '\n' + + 'File: ' + + metadataFilePath + + '\n' + + fileContent + ) + } + const hclIntegration = hclConfig.result.data.integration[0] + + // Read the README + let readmeContent: string | null = null + if (hclIntegration.docs[0].process_docs) { + const readmeFile = path.join( + integrationDirectory, + hclIntegration.docs[0].readme_location + ) + + // Throw if the README file doesn't exist + if (!fs.existsSync(readmeFile)) { + throw new Error( + `The README file, ${readmeFile}, was derived from config values and integration data, but it does not exist.` + + ` ` + + `Please double check the "readme_location" value in ${metadataFilePath}, and try again.` + ) + } + readmeContent = fs.readFileSync(readmeFile, 'utf8') + } + + // Calculate each Component object + const allComponents: Array = [] + for (let i = 0; i < hclIntegration.component.length; i++) { + allComponents.push( + await loadComponent( + integrationDirectory, + hclIntegration.component[i].type, + hclIntegration.component[i].name, + hclIntegration.component[i].slug, + variableGroupConfigs + ) + ) + } + + // Return Integration with all defaults set + return { + id: integrationID, + product: integrationProductSlug, + identifier: hclIntegration.identifier, + name: hclIntegration.name, + description: hclIntegration.description, + current_release: { + version: currentReleaseVersion, + readme: readmeContent, + components: allComponents, + }, + flags: hclIntegration.flags, + docs: hclIntegration.docs[0], + hide_versions: hclIntegration.hide_versions, + license: hclIntegration.license[0], + integration_type: hclIntegration.integration_type, + } +} + +async function loadComponent( + integrationDirectory: string, + componentType: string, + componentName: string, + componentSlug: string, + variableGroupConfigs: Array +): Promise { + // Calculate the location of the folder where the README / variables, etc reside + const componentFolder = `${integrationDirectory}/components/${componentType}/${componentSlug}` + + // Load the README if it exists + const componentReadmeFile = `${componentFolder}/README.md` + let readmeContent: string | null = null + try { + readmeContent = fs.readFileSync(componentReadmeFile, 'utf8') + } catch (err) { + // No issue, there's just no README, which is OK! + } + + // Go through each VariableGroupConfig to try see if we need to load them + const variableGroups: Array = [] + + for (let i = 0; i < variableGroupConfigs.length; i++) { + const variableGroupConfig = variableGroupConfigs[i] + const variableGroupFile = `${componentFolder}/${variableGroupConfig.filename}` + if (fs.existsSync(variableGroupFile)) { + // Load & Validate the Variable Files (parameters.hcl, outputs.hcl, etc.) + const fileContent = fs.readFileSync(variableGroupFile, 'utf8') + const hclConfig = new HCL( + fileContent, + getVariablesSchema(variableGroupConfig.stanza) + ) + if (!hclConfig.result.data) { + throw new Error(hclConfig.result.error.message) + } + + // Map the HCL File variable configuration to the Variable defaults + const variables: Array = hclConfig.result.data[ + variableGroupConfig.stanza + ].map((v) => { + return { + key: v.key, + description: v.description ? v.description : null, + type: v.type ? v.type : null, + required: typeof v.required != 'undefined' ? v.required : null, + default_value: v.default_value ? v.default_value : null, + } + }) + variableGroups.push({ + variable_group_config_id: variableGroupConfig.id, + variables, + }) + } else { + console.warn(`Variable Group File '${variableGroupFile}' not found.`) + } + } + + return { + type: componentType, + name: componentName, + slug: componentSlug, + readme: readmeContent, + variable_groups: variableGroups, + } +} diff --git a/packages/integrations-hcl/schemas/metadata.hcl.ts b/packages/integrations-hcl/strategies/default/metadata.hcl.ts similarity index 100% rename from packages/integrations-hcl/schemas/metadata.hcl.ts rename to packages/integrations-hcl/strategies/default/metadata.hcl.ts diff --git a/packages/integrations-hcl/schemas/variables.hcl.ts b/packages/integrations-hcl/strategies/default/variables.hcl.ts similarity index 100% rename from packages/integrations-hcl/schemas/variables.hcl.ts rename to packages/integrations-hcl/strategies/default/variables.hcl.ts diff --git a/packages/integrations-hcl/strategies/nomad-pack/load_pack_directory.ts b/packages/integrations-hcl/strategies/nomad-pack/load_pack_directory.ts new file mode 100644 index 00000000..73e92288 --- /dev/null +++ b/packages/integrations-hcl/strategies/nomad-pack/load_pack_directory.ts @@ -0,0 +1,74 @@ +import * as fs from 'fs' +import * as path from 'path' +import HCL from '../../lib/hcl' +import { Integration } from '../../schemas/integration' +import MetadataHCLSchema from './metadata.hcl' + +const PACK_README_LOCATION = './README.md' + +export async function loadNomadPackIntegrationDirectory( + integrationDirectory: string, + integrationID: string, + integrationProductSlug: string, + currentReleaseVersion: string +): Promise { + const metadataFilePath = path.join(integrationDirectory, 'metadata.hcl') + + // Read & Validate the Metadata file + const fileContent = fs.readFileSync(metadataFilePath, 'utf8') + const hclConfig = new HCL(fileContent, MetadataHCLSchema) + // throw a verbose error message with the filepath and contents + if (!hclConfig.result.data) { + throw new Error( + hclConfig.result.error.message + + '\n' + + 'File: ' + + metadataFilePath + + '\n' + + fileContent + ) + } + const appStanza = hclConfig.result.data.app[0] + const packStanza = hclConfig.result.data.pack[0] + const integrationStanza = hclConfig.result.data.integration[0] + + // Read the README, these are required for Packs + const readmeFile = path.join(integrationDirectory, PACK_README_LOCATION) + if (!fs.existsSync(readmeFile)) { + // Throw if the README file doesn't exist + throw new Error( + `The README file, ${readmeFile}, was not found.` + + ` ` + + `For Nomad Packs, the README file must be located at ./README.md, relative to the subdirectory of the Pack.` + ) + } + const readmeContent: string = fs.readFileSync(readmeFile, 'utf8') + + return { + id: integrationID, + product: integrationProductSlug, + identifier: integrationStanza.identifier, + name: packStanza.name, + description: packStanza.description, + current_release: { + version: currentReleaseVersion, + readme: readmeContent, + components: [], + }, + flags: integrationStanza.flags, + docs: { + process_docs: true, + readme_location: PACK_README_LOCATION, + external_url: appStanza.url, + }, + hide_versions: false, + license: { + type: null, + url: null, + }, + // This comes from the pack `integration_type` configuration in our integrations repo. + // In the event the slug there ever changes, so does this. + // https://github.com/hashicorp/integrations/blob/main/nomad/_config.hcl#L6 + integration_type: 'pack', + } +} diff --git a/packages/integrations-hcl/strategies/nomad-pack/metadata.hcl.ts b/packages/integrations-hcl/strategies/nomad-pack/metadata.hcl.ts new file mode 100644 index 00000000..13f7ce26 --- /dev/null +++ b/packages/integrations-hcl/strategies/nomad-pack/metadata.hcl.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' + +const app = z.object({ + url: z.string(), +}) + +const pack = z.object({ + name: z.string(), + description: z.string(), +}) + +const integration = z.object({ + identifier: z.string(), + flags: z.string().array().default([]), +}) + +const schema = z.object({ + app: app.array().length(1), + pack: pack.array().length(1), + integration: integration.array().length(1), +}) + +export default schema diff --git a/packages/integrations-hcl/tsconfig.json b/packages/integrations-hcl/tsconfig.json index 5c4db20c..19f11efb 100644 --- a/packages/integrations-hcl/tsconfig.json +++ b/packages/integrations-hcl/tsconfig.json @@ -8,7 +8,7 @@ // fix error TS1056: Accessors are only available when targeting ECMAScript 5 and higher. "target": "ESNext" }, - "include": ["./lib", "./schemas", "./index.ts"], + "include": ["./lib", "./schemas", "./strategies", "./index.ts"], "exclude": [ "node_modules", "**/__fixtures__/**/*",