Simplify postgres and GitHub forms (#24127)

* add grouping and collapsing fields to postgres source * add auth group to github source connector * revert postgres field order changes and adjust group of schemas field * inject group into ssh tunnel spec for postgres only, through overloaded methods * Automated Change * bump Dockerfile versions and update changelogs * bump strict encrypt version as well * fix postgres acceptance test * fix acceptance test again --------- Co-authored-by: lmossman <lmossman@users.noreply.github.com>
airbytehq · Mar 17, 2023 · fcc80cb · fcc80cb
1 parent c3247d3
commit fcc80cb
Show file tree

Hide file tree

Showing 11 changed files with 138 additions and 80 deletions.
diff --git a/...tegrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshHelpers.java b/...tegrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshHelpers.java
@@ -9,18 +9,32 @@
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.protocol.models.v0.ConnectorSpecification;
 import java.io.IOException;
+import java.util.Optional;
 
 public class SshHelpers {
 
   public static ConnectorSpecification getSpecAndInjectSsh() throws IOException {
+    return getSpecAndInjectSsh(Optional.empty());
+  }
+
+  public static ConnectorSpecification getSpecAndInjectSsh(final Optional<String> group) throws IOException {
     final ConnectorSpecification originalSpec = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class);
-    return injectSshIntoSpec(originalSpec);
+    return injectSshIntoSpec(originalSpec, group);
   }
 
   public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecification connectorSpecification) throws IOException {
+    return injectSshIntoSpec(connectorSpecification, Optional.empty());
+  }
+
+  public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecification connectorSpecification, final Optional<String> group)
+      throws IOException {
     final ConnectorSpecification originalSpec = Jsons.clone(connectorSpecification);
     final ObjectNode propNode = (ObjectNode) originalSpec.getConnectionSpecification().get("properties");
-    propNode.set("tunnel_method", Jsons.deserialize(MoreResources.readResource("ssh-tunnel-spec.json")));
+    final ObjectNode tunnelMethod = (ObjectNode) Jsons.deserialize(MoreResources.readResource("ssh-tunnel-spec.json"));
+    if (group.isPresent()) {
+      tunnelMethod.put("group", group.get());
+    }
+    propNode.set("tunnel_method", tunnelMethod);
     return originalSpec;
   }
 

diff --git a/...ions/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java b/...ions/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java
@@ -16,6 +16,7 @@
 import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
 import io.airbyte.protocol.models.v0.ConnectorSpecification;
 import java.util.List;
+import java.util.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -25,16 +26,25 @@ public class SshWrappedSource implements Source {
   private final Source delegate;
   private final List<String> hostKey;
   private final List<String> portKey;
+  private final Optional<String> sshGroup;
 
   public SshWrappedSource(final Source delegate, final List<String> hostKey, final List<String> portKey) {
     this.delegate = delegate;
     this.hostKey = hostKey;
     this.portKey = portKey;
+    this.sshGroup = Optional.empty();
+  }
+
+  public SshWrappedSource(final Source delegate, final List<String> hostKey, final List<String> portKey, final String sshGroup) {
+    this.delegate = delegate;
+    this.hostKey = hostKey;
+    this.portKey = portKey;
+    this.sshGroup = Optional.of(sshGroup);
   }
 
   @Override
   public ConnectorSpecification spec() throws Exception {
-    return SshHelpers.injectSshIntoSpec(delegate.spec());
+    return SshHelpers.injectSshIntoSpec(delegate.spec(), sshGroup);
   }
 
   @Override

diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.4.2
+LABEL io.airbyte.version=0.4.3
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json
@@ -12,6 +12,7 @@
         "description": "Choose how to authenticate to GitHub",
         "type": "object",
         "order": 0,
+        "group": "auth",
         "oneOf": [
           {
             "type": "object",

diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=2.0.3
+LABEL io.airbyte.version=2.0.4
 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=2.0.3
+LABEL io.airbyte.version=2.0.4
 LABEL io.airbyte.name=airbyte/source-postgres
diff --git a/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/...source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java
@@ -115,7 +115,7 @@ public class PostgresSource extends AbstractJdbcSource<PostgresType> implements
   private static final Set<String> INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer");
 
   public static Source sshWrappedSource() {
-    return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY);
+    return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, "security");
   }
 
   PostgresSource() {

diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json
@@ -10,7 +10,8 @@
         "title": "Host",
         "description": "Hostname of the database.",
         "type": "string",
-        "order": 0
+        "order": 0,
+        "group": "db"
       },
       "port": {
         "title": "Port",
@@ -20,13 +21,15 @@
         "maximum": 65536,
         "default": 5432,
         "examples": ["5432"],
-        "order": 1
+        "order": 1,
+        "group": "db"
       },
       "database": {
         "title": "Database Name",
         "description": "Name of the database.",
         "type": "string",
-        "order": 2
+        "order": 2,
+        "group": "db"
       },
       "schemas": {
         "title": "Schemas",
@@ -38,39 +41,47 @@
         "minItems": 0,
         "uniqueItems": true,
         "default": ["public"],
-        "order": 3
+        "order": 3,
+        "group": "db"
       },
       "username": {
         "title": "Username",
         "description": "Username to access the database.",
         "type": "string",
-        "order": 4
+        "order": 4,
+        "group": "auth"
       },
       "password": {
         "title": "Password",
         "description": "Password associated with the username.",
         "type": "string",
         "airbyte_secret": true,
-        "order": 5
+        "order": 5,
+        "group": "auth",
+        "always_show": true
       },
       "jdbc_url_params": {
         "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about <a href=\"https://jdbc.postgresql.org/documentation/head/connect.html\">JDBC URL parameters</a>.",
         "title": "JDBC URL Parameters (Advanced)",
         "type": "string",
-        "order": 6
+        "order": 6,
+        "group": "advanced"
       },
       "ssl": {
         "title": "Connect using SSL",
         "description": "Encrypt data using SSL. When activating SSL, please select one of the connection modes.",
         "type": "boolean",
         "default": false,
-        "order": 7
+        "order": 7,
+        "group": "security",
+        "always_show": true
       },
       "ssl_mode": {
         "title": "SSL Modes",
         "description": "SSL connection modes. \n  Read more <a href=\"https://jdbc.postgresql.org/documentation/head/ssl-client.html\"> in the docs</a>.",
         "type": "object",
-        "order": 7,
+        "order": 8,
+        "group": "security",
         "oneOf": [
           {
             "title": "disable",
@@ -218,7 +229,8 @@
         "type": "object",
         "title": "Replication Method",
         "description": "Replication method for extracting data from the database.",
-        "order": 8,
+        "order": 9,
+        "group": "advanced",
         "oneOf": [
           {
             "title": "Standard",
@@ -276,14 +288,33 @@
                 "type": "string",
                 "title": "LSN commit behaviour",
                 "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.",
-                "enum": ["While reading Data", "After loading Data in the destination"],
+                "enum": [
+                  "While reading Data",
+                  "After loading Data in the destination"
+                ],
                 "default": "After loading Data in the destination",
                 "order": 6
               }
             }
           }
         ]
       }
-    }
+    },
+    "groups": [
+      {
+        "id": "db"
+      },
+      {
+        "id": "auth"
+      },
+      {
+        "id": "security",
+        "title": "Security"
+      },
+      {
+        "id": "advanced",
+        "title": "Advanced"
+      }
+    ]
   }
 }
diff --git a/...rbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/...rbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java
@@ -33,6 +33,7 @@
 import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Optional;
 import org.jooq.DSLContext;
 import org.jooq.SQLDialect;
 import org.junit.jupiter.api.Test;
@@ -121,7 +122,7 @@ protected String getImageName() {
 
   @Override
   protected ConnectorSpecification getSpec() throws Exception {
-    return SshHelpers.getSpecAndInjectSsh();
+    return SshHelpers.getSpecAndInjectSsh(Optional.of("security"));
   }
 
   @Override

diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md
@@ -163,6 +163,7 @@ The GitHub connector should not run into GitHub API limitations under normal usa
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.4.3   | 2023-03-17 | [24127](https://github.com/airbytehq/airbyte/pull/24127)                                                          | Add field groups and titles to improve display of connector setup form                                                                                              |
 | 0.4.2   | 2023-03-03 | [23467](https://github.com/airbytehq/airbyte/pull/23467)                                                          | added user friendly messages, added AirbyteTracedException config_error, updated SAT                                                                                |                                                     |                                                                                                                                                                                                                                                                                                   |
 | 0.4.1   | 2023-01-27 | [22039](https://github.com/airbytehq/airbyte/pull/22039)                                                          | Set `AvailabilityStrategy` for streams explicitly to `None`                                                                                                         |                                                     |                                                                                                                                                                                                                                                                                                   |
 | 0.4.0   | 2023-01-20 | [21457](https://github.com/airbytehq/airbyte/pull/21457)                                                          | Use GraphQL for `issue_reactions` stream                                                                                                                            |