Feat: Metadata Service remove old catalog system (#26013)

* Remove Definition File References from Python files (#25590)

* Remove check_images_exist.sh

* Update definitions.py

* Update build_report.py

* Update tools/bin/ci_integration_workflow_launcher.py

* Update tools/bin/ci_check_dependency.py

* tools/bin/scan_all_spec_airbyte_secret.py

* Remove qa engine references

* Revert "Remove check_images_exist.sh"

This reverts commit 7675162.

* Improve get url function

* Add test

* remove scan_all_spec_airbyte_secret.py

* add additional test

* Remove check_images_exist.sh (#25593)

* Remove Definition File References from Java files (LocalDefinitionProvider), shell scripts and docs (#25592)

* Remove CombinedConnectorCatalogGenerator.java

* Update local definition provider

* Update local def test

* Add spec mask downloader

* Make downloader work

* Delete generators and add tests

* REMOVE THE YAML FILES

* Roughly update docs

* Update shell scripts

* Remove unused

* Add connector metadata file doc

* Apply suggestions from code review

Co-authored-by: Augustin <augustin@airbyte.io>

* Additional PR comments

* Run format tasks

---------

Co-authored-by: Augustin <augustin@airbyte.io>

* Remove unused import

* bundle registry

* Ignore future updates

* Update registry

* new registry

* Readd maskeddatainterceptor

* Automated Change

* Remove icon validation

* Automated Change

* Automated Change

* Source Amazon Ads: get rid of `fail_on_extra_columns: false` in SAT (#25913)

* Source Amazon Ads: small schema fixes

* Source Amazon Ads: update changelog

* Source Amazon Ads: update unittest

* Source Amazon Ads: unittest additional property is boolean

* Source Amazon Ads: bump version

* auto-bump connector version

---------

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>

* connectors-ci: make spec-cache / metadata bucket and creds not required for pre-release (#26119)

* Automated Change

---------

Co-authored-by: Augustin <augustin@airbyte.io>
Co-authored-by: bnchrch <bnchrch@users.noreply.github.com>
Co-authored-by: Roman Yermilov [GL] <86300758+roman-yermilov-gl@users.noreply.github.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>

.github/workflows/gradle.yml

@@ -28,17 +28,6 @@ on:
 permissions: write-all
 
 jobs:
-  # COMMON TASKS
-  ensure-images-exist:
-    name: "Ensure all required Docker images exist on Dockerhub"
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-
-      - name: Check images exist
-        run: ./tools/bin/check_images_exist.sh all
   # The output of this job is used to trigger the following builds.
   changes:
     name: "Detect Modified Files"

.github/workflows/publish-command.yml

@@ -340,27 +340,6 @@ jobs:
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
           SENTRY_ORG: airbytehq
           SENTRY_PROJECT: connector-incident-management
-      - name: Check if connector in definitions yaml
-        if: github.event.inputs.auto-bump-version == 'true' && github.event.inputs.pre-release != 'true' && success()
-        run: |
-          connector="airbyte/${{ env.IMAGE_NAME }}"
-          definitionpath=./airbyte-config-oss/init-oss/src/main/resources/seed/
-          sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml)
-          destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml)
-          if [[ (-z "$sourcecheck" && -z "$destcheck") ]]
-          then exit 1
-          fi
-      - name: Bump version in definitions yaml
-        if: github.event.inputs.auto-bump-version == 'true' && github.event.inputs.pre-release != 'true' && success()
-        run: |
-          connector="airbyte/${{ env.IMAGE_NAME }}"
-          definitionpath=./airbyte-config-oss/init-oss/src/main/resources/seed/
-          sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml)
-          destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml)
-          if [ -z "$sourcename" ]
-          then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml
-          else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml
-          fi
       - name: Run gradle process changes
         if: github.event.inputs.auto-bump-version == 'true' && github.event.inputs.pre-release != 'true' && success()
         uses: Wandalen/wretry.action@master

.pre-commit-config.yaml

@@ -33,8 +33,6 @@ repos:
         exclude: |
           (?x)^.*(
               .github/|
-              source_specs.yaml|
-              destination_specs.yaml|
               .gitlab-ci.yml
           ).?$
 

airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java

@@ -39,7 +39,7 @@ public interface FeatureFlags {
 
   /**
    * Get the workspaces allow-listed for strict incremental comparison in normalization. This takes
-   * precedence over the normalization version in destination_definitions.yaml.
+   * precedence over the normalization version in oss_registry.json .
    *
    * @return a comma-separated list of workspace ids where strict incremental comparison should be
    *         enabled in normalization.

airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java

@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.commons.logging;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.airbyte.commons.constants.AirbyteSecretConstants;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.yaml.Yamls;
+import java.nio.charset.Charset;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.io.IOUtils;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.rewrite.RewritePolicy;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.impl.Log4jLogEvent;
+import org.apache.logging.log4j.message.SimpleMessage;
+import org.apache.logging.log4j.status.StatusLogger;
+
+/**
+ * Custom Log4j2 {@link RewritePolicy} used to intercept all log messages and mask any JSON
+ * properties in the message that match the list of maskable properties.
+ * <p>
+ * The maskable properties file is generated by a Gradle task in the {@code :airbyte-config:specs}
+ * project. The file is named {@code specs_secrets_mask.yaml} and is located in the
+ * {@code src/main/resources/seed} directory of the {@link :airbyte-config:init} project.
+ */
+@Plugin(name = "MaskedDataInterceptor",
+        category = "Core",
+        elementType = "rewritePolicy",
+        printObject = true)
+public class MaskedDataInterceptor implements RewritePolicy {
+
+  protected static final Logger logger = StatusLogger.getLogger();
+
+  /**
+   * The pattern used to determine if a message contains sensitive data.
+   */
+  private final Optional<String> pattern;
+
+  @PluginFactory
+  public static MaskedDataInterceptor createPolicy(
+                                                   @PluginAttribute(value = "specMaskFile",
+                                                                    defaultString = "/seed/specs_secrets_mask.yaml") final String specMaskFile) {
+    return new MaskedDataInterceptor(specMaskFile);
+  }
+
+  private MaskedDataInterceptor(final String specMaskFile) {
+    this.pattern = buildPattern(specMaskFile);
+  }
+
+  @Override
+  public LogEvent rewrite(final LogEvent source) {
+    return Log4jLogEvent.newBuilder()
+        .setLoggerName(source.getLoggerName())
+        .setMarker(source.getMarker())
+        .setLoggerFqcn(source.getLoggerFqcn())
+        .setLevel(source.getLevel())
+        .setMessage(new SimpleMessage(applyMask(source.getMessage().getFormattedMessage())))
+        .setThrown(source.getThrown())
+        .setContextMap(source.getContextMap())
+        .setContextStack(source.getContextStack())
+        .setThreadName(source.getThreadName())
+        .setSource(source.getSource())
+        .setTimeMillis(source.getTimeMillis())
+        .build();
+  }
+
+  /**
+   * Applies the mask to the message, if necessary.
+   *
+   * @param message The log message.
+   * @return The possibly masked log message.
+   */
+  private String applyMask(final String message) {
+    if (pattern.isPresent()) {
+      return message.replaceAll(pattern.get(), "\"$1\":\"" + AirbyteSecretConstants.SECRETS_MASK + "\"");
+    } else {
+      return message;
+    }
+  }
+
+  /**
+   * Loads the maskable properties from the provided file.
+   *
+   * @param specMaskFile The spec mask file.
+   * @return The set of maskable properties.
+   */
+  private Set<String> getMaskableProperties(final String specMaskFile) {
+    // URL url = MaskedDataInterceptor.class.getResource("/");
+
+    // final URL url2 = Resources.getResource("");
+    // try {
+    // logger.info("Loading mask data from {} in class {}", url2, Paths.get(url.toURI()).toFile());
+    // } catch (final Exception e) {
+    // logger.error("Unable to load mask data from '{}': {}.", specMaskFile, e.getMessage());
+    // }
+
+    try {
+      final String maskFileContents = IOUtils.toString(getClass().getResourceAsStream(specMaskFile), Charset.defaultCharset());
+      final Map<String, Set<String>> properties = Jsons.object(Yamls.deserialize(maskFileContents), new TypeReference<>() {});
+      return properties.getOrDefault("properties", Set.of());
+    } catch (final Exception e) {
+      logger.error("Unable to load mask data from '{}': {}.", specMaskFile, e.getMessage());
+      return Set.of();
+    }
+  }
+
+  /**
+   * Builds the maskable property matching pattern.
+   *
+   * @param specMaskFile The spec mask file.
+   * @return The regular expression pattern used to find maskable properties.
+   */
+  private Optional<String> buildPattern(final String specMaskFile) {
+    final Set<String> maskableProperties = getMaskableProperties(specMaskFile);
+    return !maskableProperties.isEmpty() ? Optional.of(generatePattern(maskableProperties)) : Optional.empty();
+  }
+
+  /**
+   * Generates the property matching pattern string from the provided set of properties.
+   *
+   * @param properties The set of properties to match.
+   * @return The generated regular expression pattern used to match the maskable properties.
+   */
+  private String generatePattern(final Set<String> properties) {
+    final StringBuilder builder = new StringBuilder();
+    builder.append("(?i)"); // case insensitive
+    builder.append("\"(");
+    builder.append(properties.stream().collect(Collectors.joining("|")));
+    builder.append(")\"\\s*:\\s*(\"(?:[^\"\\\\]|\\\\.)*\"|\\[[^]\\[]*]|\\d+)");
+    return builder.toString();
+  }
+
+}

airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java

@@ -10,7 +10,7 @@ public class CatalogDefinitionsConfig {
 
   private static final String SEED_SUBDIRECTORY = "seed/";
   private static final String ICON_SUBDIRECTORY = "icons/";
-  private static final String LOCAL_CONNECTOR_CATALOG_FILE_NAME = "oss_catalog.json";
+  private static final String LOCAL_CONNECTOR_CATALOG_FILE_NAME = "oss_registry.json";
   private static final String DEFAULT_LOCAL_CONNECTOR_CATALOG_PATH =
       SEED_SUBDIRECTORY + LOCAL_CONNECTOR_CATALOG_FILE_NAME;
 

airbyte-config-oss/init-oss/build.gradle

@@ -18,10 +18,3 @@ dependencies {
 }
 
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)
-
-task validateIcons(type: JavaExec, dependsOn: [compileJava]) {
-    classpath = sourceSets.main.runtimeClasspath
-    mainClass = 'io.airbyte.configoss.init.IconValidationTask'
-}
-
-validateIcons.shouldRunAfter processResources