diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/LogsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/LogsHandlerTest.java index cbb0c8b9630..aafd4a2cd04 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/LogsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/LogsHandlerTest.java @@ -52,7 +52,7 @@ void testSchedulerLogs() { /** * This test ensures the masking file generated by - * {@link io.airbyte.config.specs.ConnectorSpecMaskGenerator} is accessible in the server. This is + * {@link io.airbyte.config.specs.ConnectorSpecMaskDownloader} is accessible in the server. This is * required to ensure the server masks connector secrets in logs. Since the logic and the generated * spec come from different modules the easiest way to test this is in a downstream consumer * application. diff --git a/airbyte-commons/build.gradle b/airbyte-commons/build.gradle index 3c4c0f9e196..8a6b8cdfcb0 100644 --- a/airbyte-commons/build.gradle +++ b/airbyte-commons/build.gradle @@ -1,5 +1,6 @@ plugins { id "java-library" + id "de.undercouch.download" version "5.4.0" } dependencies { @@ -9,4 +10,12 @@ dependencies { implementation 'com.jayway.jsonpath:json-path:2.7.0' } +task downloadSpecSecretMask(type: Download) { + src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml' + dest new File(projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml') + overwrite true +} + +tasks.processResources.dependsOn(downloadSpecSecretMask) + Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index a09a723ce35..fe996bed3f1 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -34,12 +34,6 @@ tasks.named("buildDockerImage") { Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) -task validateIcons(type: JavaExec, dependsOn: [compileJava]) { - classpath = sourceSets.main.runtimeClasspath - mainClass = 'io.airbyte.config.init.IconValidationTask' -} - -validateIcons.shouldRunAfter processResources processResources { from("${project.rootDir}/airbyte-connector-builder-resources") } \ No newline at end of file diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/IconValidationTask.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/IconValidationTask.java deleted file mode 100644 index 814b14f4f8d..00000000000 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/IconValidationTask.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.init; - -import com.google.common.io.Resources; -import io.airbyte.commons.constants.AirbyteCatalogConstants; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Simple task that checks if all icons in the seed definition files exist as well as that no icon - * in the icons folder is unused. - */ -public class IconValidationTask { - - private static Path getIconDirectoryPath() { - try { - final URI localIconsUri = Resources.getResource(AirbyteCatalogConstants.ICON_SUBDIRECTORY).toURI(); - return Path.of(localIconsUri); - } catch (final URISyntaxException e) { - throw new RuntimeException("Failed to fetch local icon directory path", e); - } - } - - private static List getLocalIconFileNames() { - try { - final Path iconDirectoryPath = getIconDirectoryPath(); - return Files.list(iconDirectoryPath).map(path -> path.getFileName().toString()).toList(); - } catch (final IOException e) { - throw new RuntimeException("Failed to fetch local icon files", e); - } - } - - private static List getIconFileNamesFromCatalog() { - final LocalDefinitionsProvider localDefinitionsProvider = new LocalDefinitionsProvider(); - final List sourceIcons = localDefinitionsProvider - .getSourceDefinitions() - .stream().map(s -> s.getIcon()) - .collect(Collectors.toList()); - - final List destinationIcons = localDefinitionsProvider - .getDestinationDefinitions() - .stream().map(s -> s.getIcon()) - .collect(Collectors.toList()); - - // concat the two lists one - sourceIcons.addAll(destinationIcons); - - // remove all null values - sourceIcons.removeAll(Collections.singleton(null)); - - return sourceIcons; - } - - private static List difference(final List list1, final List list2) { - List difference = new ArrayList<>(list1); - difference.removeAll(list2); - return difference; - } - - public static void main(final String[] args) throws Exception { - final List catalogIconFileNames = getIconFileNamesFromCatalog(); - final List localIconFileNames = getLocalIconFileNames(); - - final List missingIcons = difference(catalogIconFileNames, localIconFileNames); - final List unusedIcons = difference(localIconFileNames, catalogIconFileNames); - - final List errorMessages = List.of(); - if (!missingIcons.isEmpty()) { - errorMessages - .add("The following icon files have been referenced inside the seed files, but don't exist:\n\n" + String.join(", ", missingIcons)); - } - - if (!unusedIcons.isEmpty()) { - errorMessages.add("The following icons are not used in the seed files and should be removed:\n\n" + String.join(", ", unusedIcons)); - } - - if (!errorMessages.isEmpty()) { - throw new RuntimeException(String.join("\n\n", errorMessages)); - } - } - -} diff --git a/airbyte-config/specs/build.gradle b/airbyte-config/specs/build.gradle index 8d1dadc3f23..3a15875d6b8 100644 --- a/airbyte-config/specs/build.gradle +++ b/airbyte-config/specs/build.gradle @@ -2,6 +2,7 @@ import java.util.concurrent.TimeUnit plugins { id 'java-library' + id "de.undercouch.download" version "5.4.0" } dependencies { @@ -14,43 +15,12 @@ dependencies { implementation project(':airbyte-json-validation') } -task downloadConnectorRegistry(type: JavaExec, dependsOn: compileJava) { - /** - * run this once a day or if the file doesn't exist. if you want to force this task to run - * do so with --rerun e.g. ./gradlew :airbyte-config:specs:downloadConnectorRegistry --info --rerun - */ - def outputFile = file(rootProject.project(':airbyte-config:init').projectDir.path + '/src/main/resources/seed/oss_registry.json') - outputs.upToDateWhen { outputFile.exists() && - System.currentTimeMillis() - outputFile.lastModified() < TimeUnit.DAYS.toMillis(1) } - classpath = sourceSets.main.runtimeClasspath - mainClass = 'io.airbyte.config.specs.ConnectorRegistryDownloader' - args project(":airbyte-config:init").projectDir +task downloadConnectorRegistry(type: Download) { + src 'https://connectors.airbyte.com/files/registries/v0/oss_registry.json' + dest new File(project(":airbyte-config:init").projectDir, 'src/main/resources/seed/oss_registry.json') + overwrite true } -project(":airbyte-config:init").tasks.processResources.dependsOn(downloadConnectorRegistry) -project(":airbyte-config:init").tasks.processTestResources.dependsOn(downloadConnectorRegistry) -project(":airbyte-config:init").tasks.test.dependsOn(downloadConnectorRegistry) - - -task generateConnectorSpecsMask(type: JavaExec, dependsOn: downloadConnectorRegistry) { - classpath = sourceSets.main.runtimeClasspath - - mainClass = 'io.airbyte.config.specs.ConnectorSpecMaskGenerator' - - args '--resource-root' - args new File(project(":airbyte-config:init").projectDir, '/src/main/resources') -} - -// TODO (ben): Remove once cloud is no longer depenedant on this. -task generateSeedConnectorSpecs(type: JavaExec, dependsOn: generateConnectorSpecsMask) { - classpath = sourceSets.main.runtimeClasspath - - mainClass = 'io.airbyte.config.specs.SeedConnectorSpecGenerator' - - args '--seed-root' - args new File(project(":airbyte-config:init").projectDir, '/src/main/resources/seed') -} - -project(":airbyte-config:init").tasks.processResources.dependsOn(generateConnectorSpecsMask) +project(":airbyte-config:init").processResources.dependsOn(downloadConnectorRegistry) Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/ConnectorRegistryDownloader.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/ConnectorRegistryDownloader.java deleted file mode 100644 index 83e6c1e9284..00000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/ConnectorRegistryDownloader.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import io.airbyte.commons.constants.AirbyteCatalogConstants; -import io.airbyte.config.CatalogDefinitionsConfig; -import java.net.URL; -import java.nio.file.Path; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Download connector registry from airbytehq/airbyte repository. - */ -public class ConnectorRegistryDownloader { - - private static final Logger LOGGER = LoggerFactory.getLogger(ConnectorRegistryDownloader.class); - - /** - * This method is to create a path to the resource folder in the project. This is so that it's - * available at runtime via the getResource method. - */ - public static Path getResourcePath(final String projectPath, final String relativePath) { - return Path.of(projectPath, "src/main/resources/", relativePath); - } - - /** - * This method is to download the OSS catalog from the remote URL and save it to the local resource - * folder. - */ - public static void main(final String[] args) throws Exception { - final String projectPath = args[0]; - final String relativeWritePath = CatalogDefinitionsConfig.getLocalCatalogWritePath(); - final Path writePath = getResourcePath(projectPath, relativeWritePath); - - LOGGER.info("Downloading OSS connector registry from {} to {}", AirbyteCatalogConstants.REMOTE_OSS_CATALOG_URL, writePath); - - final int timeout = 10000; - FileUtils.copyURLToFile(new URL(AirbyteCatalogConstants.REMOTE_OSS_CATALOG_URL), writePath.toFile(), timeout, timeout); - } - -} diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/ConnectorSpecMaskGenerator.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/ConnectorSpecMaskGenerator.java deleted file mode 100644 index e7953498088..00000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/ConnectorSpecMaskGenerator.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.cli.Clis; -import io.airbyte.commons.constants.AirbyteCatalogConstants; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import io.airbyte.config.CatalogDefinitionsConfig; -import io.airbyte.config.ConnectorRegistry; -import io.airbyte.config.ConnectorRegistryDestinationDefinition; -import io.airbyte.config.ConnectorRegistrySourceDefinition; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Path; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This script is responsible for generating a set of connection configuration properties that have - * been marked as secret and therefore should be automatically masked if/when the - * configuration object is logged. - *

- * Specs are stored in a separate file from the definitions in an effort to keep the definitions - * yaml files human-readable and easily-editable, as specs can be rather large. - *

- * The generated mask file is created in the same location as the spec files provided to this - * script. - */ -public class ConnectorSpecMaskGenerator { - - private static final Logger LOGGER = LoggerFactory.getLogger(ConnectorSpecMaskGenerator.class); - - private static final String LOCAL_CONNECTOR_CATALOG_PATH = CatalogDefinitionsConfig.getLocalCatalogWritePath(); - - private static final Option RESOURCE_ROOT_OPTION = Option.builder("r").longOpt("resource-root").hasArg(true).required(true) - .desc("path to what project to pull resources from").build(); - private static final Options OPTIONS = new Options().addOption(RESOURCE_ROOT_OPTION); - - public static Path getResourcePath(final String projectPath, final String relativePath) { - return Path.of(projectPath, relativePath); - } - - public static void main(final String[] args) { - final CommandLine parsed = Clis.parse(args, OPTIONS); - final String resource = parsed.getOptionValue(RESOURCE_ROOT_OPTION.getOpt()); - final Path catalogPath = getResourcePath(resource, LOCAL_CONNECTOR_CATALOG_PATH); - final Path maskWritePath = getResourcePath(resource, AirbyteCatalogConstants.LOCAL_SECRETS_MASKS_PATH); - - LOGGER.info("Looking for catalog file at '{}'...", catalogPath); - - final File inputFile = catalogPath.toFile(); - - if (inputFile != null && inputFile.exists()) { - LOGGER.info("Found catalog for processing."); - final String jsonString = readFile(inputFile); - final ConnectorRegistry registry = Jsons.deserialize(jsonString, ConnectorRegistry.class); - final Stream destinationSpecs = - registry.getDestinations().stream().map(ConnectorRegistryDestinationDefinition::getSpec); - final Stream sourceSpecs = registry.getSources().stream().map(ConnectorRegistrySourceDefinition::getSpec); - - final Set secretPropertyNames = Stream.concat(destinationSpecs, sourceSpecs) - .map(ConnectorSpecMaskGenerator::findSecrets) - .flatMap(Set::stream) - .collect(Collectors.toCollection(TreeSet::new)); - - final String outputString = String.format("# This file is generated by %s.\n", ConnectorSpecMaskGenerator.class.getName()) - + "# Do NOT edit this file directly. See generator class for more details.\n" - + Yamls.serialize(Map.of("properties", secretPropertyNames)); - IOs.writeFile(maskWritePath, outputString); - LOGGER.info("Finished generating spec mask file '{}'.", maskWritePath); - } else { - throw new RuntimeException(String.format("No catalog files found in '%s'. Nothing to generate.", resource)); - } - } - - private static Set findSecrets(final ConnectorSpecification spec) { - final SpecMaskPropertyGenerator specMaskPropertyGenerator = new SpecMaskPropertyGenerator(); - final JsonNode properties = spec.getConnectionSpecification().get("properties"); - return specMaskPropertyGenerator.getSecretFieldNames(properties); - } - - private static String readFile(final File file) { - try { - LOGGER.info("Reading spec file '{}'...", file.getAbsolutePath()); - return FileUtils.readFileToString(file, Charset.defaultCharset()); - } catch (final IOException e) { - LOGGER.error("Unable to read contents of '{}'.", file.getAbsolutePath(), e); - return null; - } - } - -} diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java deleted file mode 100644 index 3050c895dc6..00000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorSpecGenerator.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.cloud.storage.StorageOptions; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.cli.Clis; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.commons.yaml.Yamls; -import io.airbyte.config.DockerImageSpec; -import io.airbyte.config.EnvConfigs; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This script is responsible for ensuring that up-to-date {@link ConnectorSpecification}s for every - * connector definition in the seed are stored in a corresponding resource file, for the purpose of - * seeding the specs into the config database on server startup. See - * ./airbyte-config/specs/readme.md for more details on how this class is run and how it fits into - * the project. - *

- * Specs are stored in a separate file from the definitions in an effort to keep the definitions - * yaml files human-readable and easily-editable, as specs can be rather large. - *

- * Specs are fetched from the GCS spec cache bucket, so if any specs are missing from the bucket - * then this will fail. Note that this script only pulls specs from the bucket cache; it never - * pushes specs to the bucket. Since this script runs at build time, the decision was to depend on - * the bucket cache rather than running a docker container to fetch the spec during the build which - * could be slow and unwieldy. If there is a failure, check the bucket cache and figure out how to - * get the correct spec in there. - */ -@SuppressWarnings("PMD.SignatureDeclareThrowsException") -public class SeedConnectorSpecGenerator { - - private static final String DOCKER_REPOSITORY_FIELD = "dockerRepository"; - private static final String DOCKER_IMAGE_TAG_FIELD = "dockerImageTag"; - private static final String DOCKER_IMAGE_FIELD = "dockerImage"; - private static final String SPEC_FIELD = "spec"; - private static final String SPEC_BUCKET_NAME = new EnvConfigs().getSpecCacheBucket(); - - private static final Logger LOGGER = LoggerFactory.getLogger(SeedConnectorSpecGenerator.class); - - private static final Option SEED_ROOT_OPTION = Option.builder("s").longOpt("seed-root").hasArg(true).required(true) - .desc("path to where seed resource files are stored").build(); - private static final Options OPTIONS = new Options().addOption(SEED_ROOT_OPTION); - - private final GcsBucketSpecFetcher bucketSpecFetcher; - - public SeedConnectorSpecGenerator(final GcsBucketSpecFetcher bucketSpecFetcher) { - this.bucketSpecFetcher = bucketSpecFetcher; - } - - public static void main(final String[] args) throws Exception { - final CommandLine parsed = Clis.parse(args, OPTIONS); - final Path outputRoot = Path.of(parsed.getOptionValue(SEED_ROOT_OPTION.getOpt())); - - final GcsBucketSpecFetcher bucketSpecFetcher = new GcsBucketSpecFetcher(StorageOptions.getDefaultInstance().getService(), SPEC_BUCKET_NAME); - final SeedConnectorSpecGenerator seedConnectorSpecGenerator = new SeedConnectorSpecGenerator(bucketSpecFetcher); - seedConnectorSpecGenerator.run(outputRoot, SeedConnectorType.SOURCE); - seedConnectorSpecGenerator.run(outputRoot, SeedConnectorType.DESTINATION); - } - - public void run(final Path seedRoot, final SeedConnectorType seedConnectorType) throws IOException { - LOGGER.info("Updating seeded {} definition specs if necessary...", seedConnectorType.name()); - - final JsonNode seedDefinitionsJson = yamlToJson(seedRoot, seedConnectorType.getDefinitionFileName()); - final JsonNode seedSpecsJson = yamlToJson(seedRoot, seedConnectorType.getSpecFileName()); - - final List updatedSeedSpecs = fetchUpdatedSeedSpecs(seedDefinitionsJson, seedSpecsJson); - - final String outputString = String.format("# This file is generated by %s.\n", this.getClass().getName()) - + "# Do NOT edit this file directly. See generator class for more details.\n" - + Yamls.serialize(updatedSeedSpecs); - final var outputPath = seedRoot.resolve(seedConnectorType.getSpecFileName()); - IOs.writeFile(outputPath, outputString); - - LOGGER.info("Finished updating {}", outputPath); - } - - private JsonNode yamlToJson(final Path root, final String fileName) { - final String yamlString = IOs.readFile(root.resolve(fileName)); - return Yamls.deserialize(yamlString); - } - - @VisibleForTesting - final List fetchUpdatedSeedSpecs(final JsonNode seedDefinitions, final JsonNode currentSeedSpecs) { - final List seedDefinitionsDockerImages = MoreIterators.toList(seedDefinitions.elements()) - .stream() - .map(json -> String.format("%s:%s", json.get(DOCKER_REPOSITORY_FIELD).asText(), json.get(DOCKER_IMAGE_TAG_FIELD).asText())) - .collect(Collectors.toList()); - - final Map currentSeedImageToSpec = MoreIterators.toList(currentSeedSpecs.elements()) - .stream() - .collect(Collectors.toMap( - json -> json.get(DOCKER_IMAGE_FIELD).asText(), - json -> new DockerImageSpec().withDockerImage(json.get(DOCKER_IMAGE_FIELD).asText()) - .withSpec(Jsons.object(json.get(SPEC_FIELD), ConnectorSpecification.class)))); - - return seedDefinitionsDockerImages - .stream() - .map(dockerImage -> currentSeedImageToSpec.containsKey(dockerImage) ? currentSeedImageToSpec.get(dockerImage) : fetchSpecFromGCS(dockerImage)) - .collect(Collectors.toList()); - } - - private DockerImageSpec fetchSpecFromGCS(final String dockerImage) { - LOGGER.info("Seeded spec not found for docker image {} - fetching from GCS bucket {}...", dockerImage, bucketSpecFetcher.getBucketName()); - final ConnectorSpecification spec = bucketSpecFetcher.attemptFetch(dockerImage) - .orElseThrow(() -> new RuntimeException(String.format( - "Failed to fetch valid spec file for docker image %s from GCS bucket %s. This will continue to fail until the connector change has been approved and published. See https://github.com/airbytehq/airbyte/tree/master/docs/connector-development#publishing-a-connector for more details.", - dockerImage, - bucketSpecFetcher.getBucketName()))); - return new DockerImageSpec().withDockerImage(dockerImage).withSpec(spec); - } - -} diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorType.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorType.java deleted file mode 100644 index 0c53edab074..00000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SeedConnectorType.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -/** - * Enum to handle fetching connector definitions with appropriate specs. - */ -public enum SeedConnectorType { - - SOURCE( - "source_definitions.yaml", - "source_specs.yaml"), - DESTINATION( - "destination_definitions.yaml", - "destination_specs.yaml"); - - private final String definitionFileName; - private final String specFileName; - - SeedConnectorType(final String definitionFileName, - final String specFileName) { - this.definitionFileName = definitionFileName; - this.specFileName = specFileName; - } - - public String getDefinitionFileName() { - return definitionFileName; - } - - public String getSpecFileName() { - return specFileName; - } - -} diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SpecMaskPropertyGenerator.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SpecMaskPropertyGenerator.java deleted file mode 100644 index 3ab3fc4f7ef..00000000000 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/SpecMaskPropertyGenerator.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.constants.AirbyteSecretConstants; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.Set; - -/** - * Generates a set of property names from the provided connection spec properties object that are - * marked as secret. - */ -public class SpecMaskPropertyGenerator { - - /** - * Builds a set of property names from the provided connection spec properties object that are - * marked as secret. - * - * @param properties The connection spec properties. - * @return A set of property names that have been marked as secret. - */ - public Set getSecretFieldNames(final JsonNode properties) { - final Set secretPropertyNames = new HashSet<>(); - if (properties != null && properties.isObject()) { - final Iterator> fields = properties.fields(); - while (fields.hasNext()) { - final Entry field = fields.next(); - - /* - * If the current field is an object, check if it represents a secret. If it does, add it to the set - * of property names. If not, recursively call this method again with the field value to see if it - * contains any secrets. - * - * If the current field is an array, recursively call this method for each field within the value to - * see if any of those contain any secrets. - */ - if (field.getValue().isObject()) { - if (field.getValue().has(AirbyteSecretConstants.AIRBYTE_SECRET_FIELD)) { - if (field.getValue().get(AirbyteSecretConstants.AIRBYTE_SECRET_FIELD).asBoolean()) { - secretPropertyNames.add(field.getKey()); - } - } else { - secretPropertyNames.addAll(getSecretFieldNames(field.getValue())); - } - } else if (field.getValue().isArray()) { - for (int i = 0; i < field.getValue().size(); i++) { - secretPropertyNames.addAll(getSecretFieldNames(field.getValue().get(i))); - } - } - } - } - - return secretPropertyNames; - } - -} diff --git a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/ConnectorSpecMaskGeneratorTest.java b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/ConnectorSpecMaskGeneratorTest.java deleted file mode 100644 index be63eb212e6..00000000000 --- a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/ConnectorSpecMaskGeneratorTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import static io.airbyte.commons.constants.AirbyteCatalogConstants.LOCAL_SECRETS_MASKS_PATH; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.Set; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.Test; - -/** - * Test suite for the {@link ConnectorSpecMaskGenerator} class. - */ -class ConnectorSpecMaskGeneratorTest { - - @Test - void testConnectorSpecMaskGenerator() throws IOException { - final String directory = "src/test/resources/valid_specs"; - final File outputFile = new File(directory, LOCAL_SECRETS_MASKS_PATH); - final String[] args = {"--resource-root", directory}; - - ConnectorSpecMaskGenerator.main(args); - assertTrue(outputFile.exists()); - - final JsonNode maskContents = Yamls.deserialize(FileUtils.readFileToString(outputFile, Charset.defaultCharset())); - assertEquals(Set.of("azure_blob_storage_account_key", "api_key"), - Jsons.object(maskContents.get("properties"), new TypeReference>() {})); - } - - @Test - void testConnectorSpecMaskGeneratorNoSpecs() { - final String directory = "src/test/resources/no_specs"; - final String[] args = {"--resource-root", directory}; - assertThrows(RuntimeException.class, () -> ConnectorSpecMaskGenerator.main(args)); - } - -} diff --git a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SpecMaskPropertyGeneratorTest.java b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SpecMaskPropertyGeneratorTest.java deleted file mode 100644 index 3433d7dffd4..00000000000 --- a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/SpecMaskPropertyGeneratorTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.specs; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import java.util.Set; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * Test suite for the {@link SpecMaskPropertyGenerator} class. - */ -@SuppressWarnings("LineLength") -class SpecMaskPropertyGeneratorTest { - - private SpecMaskPropertyGenerator specMaskPropertyGenerator; - - @BeforeEach - void setup() { - specMaskPropertyGenerator = new SpecMaskPropertyGenerator(); - } - - @Test - void testSecretProperties() { - final JsonNode json = Jsons.deserialize( - "{\"api_key\":{\"type\":\"string\",\"description\":\"The API Key for the Airtable account.\",\"title\":\"API Key\",\"airbyte_secret\":true,\"examples\":[\"key1234567890\"],\"base_id\":{\"type\":\"string\",\"description\":\"The Base ID to integrate the data from.\",\"title\":\"Base ID\",\"examples\":[\"app1234567890\"]},\"tables\":{\"type\":\"array\",\"items\":[{\"type\":\"string\"}],\"description\":\"The list of Tables to integrate.\",\"title\":\"Tables\",\"examples\":[\"table 1\",\"table 2\"]}}}"); - final Set propertyNames = specMaskPropertyGenerator.getSecretFieldNames(json); - assertEquals(Set.of("api_key"), propertyNames); - } - - @Test - void testSecretPropertiesFalse() { - final JsonNode json = Jsons.deserialize( - "{\"api_key\":{\"type\":\"string\",\"description\":\"The API Key for the Airtable account.\",\"title\":\"API Key\",\"airbyte_secret\":false,\"examples\":[\"key1234567890\"],\"base_id\":{\"type\":\"string\",\"description\":\"The Base ID to integrate the data from.\",\"title\":\"Base ID\",\"examples\":[\"app1234567890\"]},\"tables\":{\"type\":\"array\",\"items\":[{\"type\":\"string\"}],\"description\":\"The list of Tables to integrate.\",\"title\":\"Tables\",\"examples\":[\"table 1\",\"table 2\"]}}}"); - final Set propertyNames = specMaskPropertyGenerator.getSecretFieldNames(json); - assertEquals(0, propertyNames.size()); - } - - @Test - void testNestedSecretProperties() { - final JsonNode json = Jsons.deserialize( - "{\"title\":\"Authentication Method\",\"type\":\"object\",\"description\":\"The type of authentication to be used\",\"oneOf\":[{\"title\":\"None\",\"additionalProperties\":false,\"description\":\"No authentication will be used\",\"required\":[\"method\"],\"properties\":{\"method\":{\"type\":\"string\",\"const\":\"none\"}}},{\"title\":\"Api Key/Secret\",\"additionalProperties\":false,\"description\":\"Use a api key and secret combination to authenticate\",\"required\":[\"method\",\"apiKeyId\",\"apiKeySecret\"],\"properties\":{\"method\":{\"type\":\"string\",\"const\":\"secret\"},\"apiKeyId\":{\"title\":\"API Key ID\",\"description\":\"The Key ID to used when accessing an enterprise Elasticsearch instance.\",\"type\":\"string\"},\"apiKeySecret\":{\"title\":\"API Key Secret\",\"description\":\"The secret associated with the API Key ID.\",\"type\":\"string\",\"airbyte_secret\":true}}},{\"title\":\"Username/Password\",\"additionalProperties\":false,\"description\":\"Basic auth header with a username and password\",\"required\":[\"method\",\"username\",\"password\"],\"properties\":{\"method\":{\"type\":\"string\",\"const\":\"basic\"},\"username\":{\"title\":\"Username\",\"description\":\"Basic auth username to access a secure Elasticsearch server\",\"type\":\"string\"},\"password\":{\"title\":\"Password\",\"description\":\"Basic auth password to access a secure Elasticsearch server\",\"type\":\"string\",\"airbyte_secret\":true}}}]}"); - final Set propertyNames = specMaskPropertyGenerator.getSecretFieldNames(json); - assertEquals(Set.of("apiKeySecret", "password"), propertyNames); - } - - @Test - void testNullProperties() { - final Set propertyNames = specMaskPropertyGenerator.getSecretFieldNames(null); - assertEquals(0, propertyNames.size()); - } - - @Test - void testNonObjectProperties() { - final JsonNode json = Jsons.deserialize("{\"array\":[\"foo\",\"bar\"]}"); - final Set propertyNames = specMaskPropertyGenerator.getSecretFieldNames(json.get("array")); - assertEquals(0, propertyNames.size()); - } - -} diff --git a/build.gradle b/build.gradle index a340fb23cd5..62872982881 100644 --- a/build.gradle +++ b/build.gradle @@ -124,7 +124,8 @@ def createSpotlessTarget = { pattern -> 'secrets', 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. 'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. - 'resources/seed/*_catalog.json', // Do not remove - this is also necessary to prevent diffs in our github workflows + 'resources/seed/specs_secrets_mask.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. + 'resources/seed/*_registry.json', // Do not remove - this is also necessary to prevent diffs in our github workflows 'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad 'airbyte-webapp', // The webapp module uses its own auto-formatter, so spotless is not necessary here 'airbyte-connector-builder-server/connector_builder/generated', // autogenerated code doesn't need to be formatted diff --git a/tools/bin/build_report.py b/tools/bin/build_report.py deleted file mode 100644 index 333bfaeec83..00000000000 --- a/tools/bin/build_report.py +++ /dev/null @@ -1,260 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -""" -Report Connector Build Status to Slack - -All invocations of this script must be run from the Airbyte repository root. - -BEFORE RUNNING THIS SCRIPT: -- Ensure you have read the documentation on how this system works: https://internal-docs.airbyte.io/Generated-Reports/Build-Status-Reports - -To Run tests: -pytest ./tools/bin/build_report.py - -To run the script: -pip install slack-sdk pyyaml -python ./tools/bin/build_report.py -""" - -import os -import pathlib -import re -import sys -from typing import Dict, List, Optional - -import requests -import yaml -from slack_sdk import WebhookClient -from slack_sdk.errors import SlackApiError - -# Global statics -CONNECTOR_DEFINITIONS_DIR = "./airbyte-config/init/src/main/resources/seed" -SOURCE_DEFINITIONS_YAML = f"{CONNECTOR_DEFINITIONS_DIR}/source_definitions.yaml" -DESTINATION_DEFINITIONS_YAML = f"{CONNECTOR_DEFINITIONS_DIR}/destination_definitions.yaml" -CONNECTORS_ROOT_PATH = "./airbyte-integrations/connectors" -RELEVANT_BASE_MODULES = ["base-normalization", "connector-acceptance-test"] -CONNECTOR_BUILD_OUTPUT_URL = "https://dnsgjos7lj2fu.cloudfront.net/tests/summary/connectors" - -# Global vars -TESTED_SOURCE = [] -TESTED_DESTINATION = [] -SUCCESS_SOURCE = [] -SUCCESS_DESTINATION = [] -NO_TESTS = [] -FAILED_LAST = [] -FAILED_2_LAST = [] - - -def get_status_page(connector) -> str: - response = requests.get(f"{CONNECTOR_BUILD_OUTPUT_URL}/{connector}/index.html") - if response.status_code == 200: - return response.text - - -def parse(page) -> list: - history = [] - for row in re.findall(r"(.*?)", page): - cols = re.findall(r"(.*?)", row) - if not cols or len(cols) != 3: - continue - history.append( - { - "date": cols[0], - "status": re.findall(r" (\S+)", cols[1])[0], - "link": re.findall(r'href="(.*?)"', cols[2])[0], - } - ) - return history - - -def check_module(connector): - status_page = get_status_page(connector) - - # check if connector is tested - if not status_page: - NO_TESTS.append(connector) - print("F", end="", flush=True) - return - - print(".", end="", flush=True) - - if connector.startswith("source"): - TESTED_SOURCE.append(connector) - elif connector.startswith("destination"): - TESTED_DESTINATION.append(connector) - - # order: recent values goes first - history = parse(status_page) - # order: recent values goes last - short_status = "".join(["✅" if build["status"] == "success" else "❌" for build in history[::-1]]) # ex: ❌✅✅❌✅✅❌❌ - - # check latest build status - last_build = history[0] - if last_build["status"] == "success": - if connector.startswith("source"): - SUCCESS_SOURCE.append(connector) - elif connector.startswith("destination"): - SUCCESS_DESTINATION.append(connector) - else: - failed_today = [connector, short_status, last_build["link"], last_build["date"]] - - if len(history) > 1 and history[1]["status"] != "success": - FAILED_2_LAST.append(failed_today) - return - - FAILED_LAST.append(failed_today) - - -def failed_report(failed_report) -> str: - max_name_len = max([len(connector[0]) for connector in failed_report]) - max_status_len = max(len(connector[1]) for connector in failed_report) - for connector in failed_report: - connector[0] = connector[0].ljust(max_name_len, " ") - connector[1] = connector[1].rjust(max_status_len, " ") - return "\n".join([" ".join(connector) for connector in failed_report]) - - -def create_report(connectors, statuses: List[str]) -> str: - sources_len = len([name for name in connectors if name.startswith("source")]) - destinations_len = len([name for name in connectors if name.startswith("destination")]) - - report = f""" -CONNECTORS: total: {len(connectors)} {" & ".join(statuses)} connectors -Sources: total: {sources_len} / tested: {len(TESTED_SOURCE)} / success: {len(SUCCESS_SOURCE)} ({round(len(SUCCESS_SOURCE) / sources_len * 100, 1)}%) -Destinations: total: {destinations_len} / tested: {len(TESTED_DESTINATION)} / success: {len(SUCCESS_DESTINATION)} ({round(len(SUCCESS_DESTINATION) / destinations_len * 100, 1)}%) - -""" - if FAILED_LAST: - report += f"FAILED LAST BUILD ONLY - {len(FAILED_LAST)} connectors:\n" + failed_report(FAILED_LAST) + "\n\n" - - if FAILED_2_LAST: - report += f"FAILED TWO LAST BUILDS - {len(FAILED_2_LAST)} connectors:\n" + failed_report(FAILED_2_LAST) + "\n\n" - - if NO_TESTS: - report += f"NO TESTS - {len(NO_TESTS)} connectors:\n" + "\n".join(NO_TESTS) + "\n" - - return report - - -def send_report(report): - webhook = WebhookClient(os.environ["SLACK_BUILD_REPORT"]) - try: - - def chunk_messages(report): - """split report into messages with no more than 4000 chars each (slack limitation)""" - msg = "" - for line in report.splitlines(): - msg += line + "\n" - if len(msg) > 3500: - yield msg - msg = "" - yield msg - - for msg in chunk_messages(report): - webhook.send(text=f"```{msg}```") - print("Report has been sent") - except SlackApiError as e: - print("Unable to send report") - assert e.response["error"] - - -def parse_dockerfile_repository_label(dockerfile_contents: str) -> Optional[str]: - parsed_label = re.findall(r"LABEL io.airbyte.name=(.*)[\s\n]*", dockerfile_contents) - if len(parsed_label) == 1: - return parsed_label[0] - elif len(parsed_label) == 0: - return None - else: - raise Exception(f"found more than one label in dockerfile: {dockerfile_contents}") - - -def get_docker_label_to_connector_directory(base_directory: str, connector_module_names: List[str]) -> Dict[str, str]: - result = {} - for connector in connector_module_names: - # parse the dockerfile label if the dockerfile exists - dockerfile_path = pathlib.Path(base_directory, connector, "Dockerfile") - if os.path.isfile(dockerfile_path): - print(f"Reading {dockerfile_path}") - with open(dockerfile_path, "r") as file: - dockerfile_contents = file.read() - label = parse_dockerfile_repository_label(dockerfile_contents) - if label: - result[label] = connector - else: - print(f"Couldn't find a connector label in {dockerfile_path}") - else: - print(f"Couldn't find a dockerfile at {dockerfile_path}") - return result - - -def get_connectors_with_release_stage(definitions_yaml: List, stages: List[str]) -> List[str]: - """returns e.g: ['airbyte/source-salesforce', ...] when given 'generally_available' as input""" - return [definition["dockerRepository"] for definition in definitions_yaml if definition.get("releaseStage", "alpha") in stages] - - -def read_definitions_yaml(path: str): - with open(path, "r") as file: - return yaml.safe_load(file) - - -def get_connectors_with_release_stages(base_directory: str, connectors: List[str], relevant_stages=["beta", "generally_available"]): - # TODO currently this also excludes shared libs like source-jdbc, we probably shouldn't do that, so we can get the build status of those - # modules as well. - connector_label_to_connector_directory = get_docker_label_to_connector_directory(base_directory, connectors) - - connectors_with_desired_status = get_connectors_with_release_stage( - read_definitions_yaml(SOURCE_DEFINITIONS_YAML), relevant_stages - ) + get_connectors_with_release_stage(read_definitions_yaml(DESTINATION_DEFINITIONS_YAML), relevant_stages) - # return appropriate directory names - return [ - connector_label_to_connector_directory[label] - for label in connectors_with_desired_status - if label in connector_label_to_connector_directory - ] - - -def setup_module(): - global pytest - global mock - - -if __name__ == "__main__": - - # find all connectors and filter to beta and GA - connectors = sorted(os.listdir(CONNECTORS_ROOT_PATH)) - relevant_stages = ["beta", "generally_available"] - relevant_connectors = get_connectors_with_release_stages(CONNECTORS_ROOT_PATH, connectors, relevant_stages) - print(f"Checking {len(relevant_connectors)} relevant connectors out of {len(connectors)} total connectors") - - # analyse build results for each connector - [check_module(connector) for connector in relevant_connectors] - [check_module(base) for base in RELEVANT_BASE_MODULES] - - report = create_report(relevant_connectors, relevant_stages) - print(report) - send_report(report) - print("Finish") -elif "pytest" in sys.argv[0]: - import unittest - - class Tests(unittest.TestCase): - def test_filter_definitions_yaml(self): - mock_def_yaml = [ - {"releaseStage": "alpha", "dockerRepository": "alpha_connector"}, - {"releaseStage": "beta", "dockerRepository": "beta_connector"}, - {"releaseStage": "generally_available", "dockerRepository": "GA_connector"}, - ] - assert ["alpha_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["alpha"]) - assert ["alpha_connector", "beta_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["alpha", "beta"]) - assert ["beta_connector", "GA_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["beta", "generally_available"]) - assert ["GA_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["generally_available"]) - - def test_parse_dockerfile_label(self): - mock_dockerfile = """ -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.8 -LABEL io.airbyte.name=airbyte/source-salesforce""" - assert "airbyte/source-salesforce" == parse_dockerfile_repository_label(mock_dockerfile) diff --git a/tools/bin/ci_check_dependency.py b/tools/bin/ci_check_dependency.py deleted file mode 100644 index ba7d0b7684f..00000000000 --- a/tools/bin/ci_check_dependency.py +++ /dev/null @@ -1,249 +0,0 @@ -import sys -import os -import os.path -import yaml -import re -from typing import Any, Dict, Text, List - -CONNECTORS_PATH = "./airbyte-integrations/connectors/" -NORMALIZATION_PATH = "./airbyte-integrations/bases/base-normalization/" -DOC_PATH = "docs/integrations/" -SOURCE_DEFINITIONS_PATH = "./airbyte-config/init/src/main/resources/seed/source_definitions.yaml" -DESTINATION_DEFINITIONS_PATH = "./airbyte-config/init/src/main/resources/seed/destination_definitions.yaml" -IGNORE_LIST = [ - # Java - "/src/test/","/src/test-integration/", "/src/testFixtures/", - # Python - "/integration_tests/", "/unit_tests/", - # Common - "acceptance-test-config.yml", "acceptance-test-docker.sh", ".md", ".dockerignore", ".gitignore", "requirements.txt"] -IGNORED_SOURCES = [ - re.compile("^source-e2e-test-cloud$"), - re.compile("^source-mongodb$"), - re.compile("^source-python-http-tutorial$"), - re.compile("^source-relational-db$"), - re.compile("^source-stock-ticker-api-tutorial$"), - re.compile("source-jdbc$"), - re.compile("^source-scaffold-.*$"), - re.compile(".*-secure$"), -] -IGNORED_DESTINATIONS = [ - re.compile(".*-strict-encrypt$"), - re.compile("^destination-dev-null$"), - re.compile("^destination-scaffold-destination-python$"), - re.compile("^destination-jdbc$") -] -COMMENT_TEMPLATE_PATH = ".github/comment_templates/connector_dependency_template.md" - - -def main(): - # Used git diff checks airbyte-integrations/ folder only - # See .github/workflows/report-connectors-dependency.yml file - git_diff_file_path = ' '.join(sys.argv[1:]) - - if git_diff_file_path == None or git_diff_file_path == "": - raise Exception("No changefile provided") - - # Get changed files - changed_files = get_changed_files(git_diff_file_path) - # Get changed modules. e.g. connector-acceptance-test from airbyte-integrations/bases/ - # or destination-mysql from airbyte-integrations/connectors/ - changed_modules = get_changed_modules(changed_files) - - # Get all existing connectors - all_connectors = get_all_connectors() - - # Getting all build.gradle file - build_gradle_files = {} - for connector in all_connectors: - connector_path = CONNECTORS_PATH + connector + "/" - build_gradle_files.update(get_gradle_file_for_path(connector_path)) - build_gradle_files.update(get_gradle_file_for_path(NORMALIZATION_PATH)) - - # Try to find dependency in build.gradle file - dependent_modules = list(set(get_dependent_modules(changed_modules, build_gradle_files))) - - # Create comment body to post on pull request - if dependent_modules: - write_report(dependent_modules) - - -def get_changed_files(path): - changed_connectors_files = [] - with open(path) as file: - for line in file: - changed_connectors_files.append(line) - return changed_connectors_files - - -def get_changed_modules(changed_files): - changed_modules = [] - for changed_file in changed_files: - # Check if this file should be ignored - if not any(ignor in changed_file for ignor in IGNORE_LIST): - split_changed_file = changed_file.split("/") - changed_modules.append(split_changed_file[1] + ":" + split_changed_file[2]) - return list(set(changed_modules)) - - -def get_all_connectors(): - walk = os.walk(CONNECTORS_PATH) - return [connector for connector in next(walk)[1]] - - -def get_gradle_file_for_path(path: str) -> Dict[Text, Any]: - if not path.endswith("/"): - path = path + "/" - build_gradle_file = find_file("build.gradle", path) - module = path.split("/")[-2] - return {module: build_gradle_file} - - -def find_file(name, path): - for root, dirs, files in os.walk(path): - if name in files: - return os.path.join(root, name) - - -def get_dependent_modules(changed_modules, all_build_gradle_files): - dependent_modules = [] - for changed_module in changed_modules: - for module, gradle_file in all_build_gradle_files.items(): - if gradle_file is None: - continue - with open(gradle_file) as file: - if changed_module in file.read(): - dependent_modules.append(module) - return dependent_modules - - -def get_connector_version(connector): - with open(f"{CONNECTORS_PATH}/{connector}/Dockerfile") as f: - for line in f: - if "io.airbyte.version" in line: - return line.split("=")[1].strip() - - -def get_connector_version_status(connector, version): - if "strict-encrypt" not in connector: - return f"`{version}`" - if connector == "source-mongodb-strict-encrypt": - base_variant_version = get_connector_version("source-mongodb-v2") - else: - base_variant_version = get_connector_version(connector.replace("-strict-encrypt", "")) - if base_variant_version == version: - return f"`{version}`" - else: - return f"❌ `{version}`
(mismatch: `{base_variant_version}`)" - - -def get_connector_changelog_status(connector: str, version) -> str: - type, name = connector.replace("-strict-encrypt", "").replace("-denormalized", "").split("-", 1) - doc_path = f"{DOC_PATH}{type}s/{name}.md" - - if any(regex.match(connector) for regex in IGNORED_SOURCES): - return "🔵
(ignored)" - if any(regex.match(connector) for regex in IGNORED_DESTINATIONS): - return "🔵
(ignored)" - if not os.path.exists(doc_path): - return "⚠
(doc not found)" - - with open(doc_path) as f: - after_changelog = False - for line in f: - if "# changelog" in line.lower(): - after_changelog = True - if after_changelog and version in line: - return "✅" - - return "❌
(changelog missing)" - - -def as_bulleted_markdown_list(items): - text = "" - for item in items: - text += f"- {item}\n" - return text - - -def as_markdown_table_rows(connectors: List[str], definitions) -> str: - text = "" - for connector in connectors: - version = get_connector_version(connector) - version_status = get_connector_version_status(connector, version) - changelog_status = get_connector_changelog_status(connector, version) - definition = next((x for x in definitions if x["dockerRepository"].endswith(connector)), None) - if any(regex.match(connector) for regex in IGNORED_SOURCES): - publish_status = "🔵
(ignored)" - elif any(regex.match(connector) for regex in IGNORED_DESTINATIONS): - publish_status = "🔵
(ignored)" - elif definition is None: - publish_status = "⚠
(not in seed)" - elif definition["dockerImageTag"] == version: - publish_status = "✅" - else: - publish_status = "❌
(diff seed version)" - text += f"| `{connector}` | {version_status} | {changelog_status} | {publish_status} |\n" - return text - - -def get_status_summary(rows: str) -> str: - if "❌" in rows: - return "❌" - elif "⚠" in rows: - return "⚠" - else: - return "✅" - - -def write_report(depended_connectors): - affected_sources = [] - affected_destinations = [] - affected_others = [] - for depended_connector in depended_connectors: - if depended_connector.startswith("source"): - affected_sources.append(depended_connector) - elif depended_connector.startswith("destination"): - affected_destinations.append(depended_connector) - else: - affected_others.append(depended_connector) - - with open(COMMENT_TEMPLATE_PATH, "r") as f: - template = f.read() - - with open(SOURCE_DEFINITIONS_PATH, 'r') as stream: - source_definitions = yaml.safe_load(stream) - with open(DESTINATION_DEFINITIONS_PATH, 'r') as stream: - destination_definitions = yaml.safe_load(stream) - - affected_sources.sort() - affected_destinations.sort() - affected_others.sort() - - source_rows = as_markdown_table_rows(affected_sources, source_definitions) - destination_rows = as_markdown_table_rows(affected_destinations, destination_definitions) - - other_status_summary = "✅" if len(affected_others) == 0 else "👀" - source_status_summary = get_status_summary(source_rows) - destination_status_summary = get_status_summary(destination_rows) - - comment = template.format( - source_open="open" if source_status_summary == "❌" else "closed", - destination_open="open" if destination_status_summary == "❌" else "closed", - source_status_summary=source_status_summary, - destination_status_summary=destination_status_summary, - other_status_summary=other_status_summary, - source_rows=source_rows, - destination_rows=destination_rows, - others_rows=as_bulleted_markdown_list(affected_others), - num_sources=len(affected_sources), - num_destinations=len(affected_destinations), - num_others=len(affected_others), - ) - - with open("comment_body.md", "w") as f: - f.write(comment) - - -if __name__ == "__main__": - main()