diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index fbaa8c6a06887..b190076e00a87 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -1771,7 +1771,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/WorkspaceIdRequestBody" + $ref: "#/components/schemas/WebBackendConnectionListRequestBody" required: true responses: "200": @@ -2738,6 +2738,24 @@ components: type: string icon: type: string + SourceSnippetRead: + type: object + required: + - sourceId + - name + - sourceDefinitionId + - sourceName + properties: + sourceId: + $ref: "#/components/schemas/SourceId" + name: + type: string + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" + sourceName: + type: string + icon: + type: string SourceReadList: type: object required: @@ -3035,6 +3053,24 @@ components: type: string icon: type: string + DestinationSnippetRead: + type: object + required: + - destinationId + - name + - destinationDefinitionId + - destinationName + properties: + destinationId: + $ref: "#/components/schemas/DestinationId" + name: + type: string + destinationDefinitionId: + $ref: "#/components/schemas/DestinationDefinitionId" + destinationName: + type: string + icon: + type: string DestinationReadList: type: object required: @@ -4624,14 +4660,23 @@ components: type: integer sourceDefinitions: type: integer + WebBackendConnectionListRequestBody: + type: object + required: + - workspaceId + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + sourceId: + $ref: "#/components/schemas/SourceId" + destinationId: + $ref: "#/components/schemas/DestinationId" WebBackendConnectionListItem: type: object description: Information about a connection that shows up in the connection list view. required: - connectionId - name - - sourceId - - destinationId - source - destination - status @@ -4642,10 +4687,6 @@ components: $ref: "#/components/schemas/ConnectionId" name: type: string - sourceId: - $ref: "#/components/schemas/SourceId" - destinationId: - $ref: "#/components/schemas/DestinationId" scheduleType: $ref: "#/components/schemas/ConnectionScheduleType" scheduleData: @@ -4653,9 +4694,9 @@ components: status: $ref: "#/components/schemas/ConnectionStatus" source: - $ref: "#/components/schemas/SourceRead" + $ref: "#/components/schemas/SourceSnippetRead" destination: - $ref: "#/components/schemas/DestinationRead" + $ref: "#/components/schemas/DestinationSnippetRead" latestSyncJobCreatedAt: $ref: "#/components/schemas/JobCreatedAt" latestSyncJobStatus: diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 614cd66a52a35..cc896bfdb0ba3 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -58,6 +58,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.validation.json.JsonValidationException; +import jakarta.annotation.Nonnull; import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; @@ -92,6 +93,8 @@ "OptionalUsedAsFieldOrParameterType"}) public class ConfigRepository { + public record StandardSyncQuery(@Nonnull UUID workspaceId, UUID sourceId, UUID destinationId, boolean includeDeleted) {} + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigRepository.class); private static final String OPERATION_IDS_AGG_FIELD = "operation_ids_agg"; private static final String OPERATION_IDS_AGG_DELIMITER = ","; @@ -843,6 +846,10 @@ public List listStandardSyncsUsingOperation(final UUID operationId } public List listWorkspaceStandardSyncs(final UUID workspaceId, final boolean includeDeleted) throws IOException { + return listWorkspaceStandardSyncs(new StandardSyncQuery(workspaceId, null, null, includeDeleted)); + } + + public List listWorkspaceStandardSyncs(final StandardSyncQuery standardSyncQuery) throws IOException { final Result connectionAndOperationIdsResult = database.query(ctx -> ctx // SELECT connection.* plus the connection's associated operationIds as a concatenated list .select( @@ -856,8 +863,10 @@ public List listWorkspaceStandardSyncs(final UUID workspaceId, fin // join with source actors so that we can filter by workspaceId .join(ACTOR).on(CONNECTION.SOURCE_ID.eq(ACTOR.ID)) - .where(ACTOR.WORKSPACE_ID.eq(workspaceId) - .and(includeDeleted ? noCondition() : CONNECTION.STATUS.notEqual(StatusType.deprecated))) + .where(ACTOR.WORKSPACE_ID.eq(standardSyncQuery.workspaceId) + .and(standardSyncQuery.destinationId == null ? noCondition() : CONNECTION.DESTINATION_ID.eq(standardSyncQuery.destinationId)) + .and(standardSyncQuery.sourceId == null ? noCondition() : CONNECTION.SOURCE_ID.eq(standardSyncQuery.sourceId)) + .and(standardSyncQuery.includeDeleted ? noCondition() : CONNECTION.STATUS.notEqual(StatusType.deprecated))) // group by connection.id so that the groupConcat above works .groupBy(CONNECTION.ID)).fetch(); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 41673ef011ccb..5ce1ff79d6fa2 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -31,6 +31,7 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.persistence.ConfigRepository.DestinationAndDefinition; import io.airbyte.config.persistence.ConfigRepository.SourceAndDefinition; +import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.protocol.models.AirbyteCatalog; @@ -188,7 +189,45 @@ void testSimpleInsertActorCatalog() throws IOException, JsonValidationException, @Test void testListWorkspaceStandardSyncAll() throws IOException { final List expectedSyncs = MockData.standardSyncs().subList(0, 4); - final List actualSyncs = configRepository.listWorkspaceStandardSyncs(MockData.standardWorkspaces().get(0).getWorkspaceId(), true); + final List actualSyncs = configRepository.listWorkspaceStandardSyncs( + MockData.standardWorkspaces().get(0).getWorkspaceId(), true); + + assertSyncsMatch(expectedSyncs, actualSyncs); + } + + @Test + void testListWorkspaceStandardSyncWithAllFiltering() throws IOException { + final UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId(); + final StandardSyncQuery query = new StandardSyncQuery(workspaceId, MockData.SOURCE_ID_1, MockData.DESTINATION_ID_1, false); + final List expectedSyncs = MockData.standardSyncs().subList(0, 3).stream() + .filter(sync -> sync.getDestinationId().equals(query.destinationId())) + .filter(sync -> sync.getSourceId().equals(query.sourceId())) + .toList(); + final List actualSyncs = configRepository.listWorkspaceStandardSyncs(query); + + assertSyncsMatch(expectedSyncs, actualSyncs); + } + + @Test + void testListWorkspaceStandardSyncDestinationFiltering() throws IOException { + final UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId(); + final StandardSyncQuery query = new StandardSyncQuery(workspaceId, null, MockData.DESTINATION_ID_1, false); + final List expectedSyncs = MockData.standardSyncs().subList(0, 3).stream() + .filter(sync -> sync.getDestinationId().equals(query.destinationId())) + .toList(); + final List actualSyncs = configRepository.listWorkspaceStandardSyncs(query); + + assertSyncsMatch(expectedSyncs, actualSyncs); + } + + @Test + void testListWorkspaceStandardSyncSourceFiltering() throws IOException { + final UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId(); + final StandardSyncQuery query = new StandardSyncQuery(workspaceId, MockData.SOURCE_ID_2, null, false); + final List expectedSyncs = MockData.standardSyncs().subList(0, 3).stream() + .filter(sync -> sync.getSourceId().equals(query.sourceId())) + .toList(); + final List actualSyncs = configRepository.listWorkspaceStandardSyncs(query); assertSyncsMatch(expectedSyncs, actualSyncs); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java index 15e35c927356a..a6117a08af06e 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -79,9 +79,9 @@ public class MockData { public static final UUID SOURCE_ID_1 = UUID.randomUUID(); public static final UUID SOURCE_ID_2 = UUID.randomUUID(); private static final UUID SOURCE_ID_3 = UUID.randomUUID(); - private static final UUID DESTINATION_ID_1 = UUID.randomUUID(); - private static final UUID DESTINATION_ID_2 = UUID.randomUUID(); - private static final UUID DESTINATION_ID_3 = UUID.randomUUID(); + public static final UUID DESTINATION_ID_1 = UUID.randomUUID(); + public static final UUID DESTINATION_ID_2 = UUID.randomUUID(); + public static final UUID DESTINATION_ID_3 = UUID.randomUUID(); private static final UUID OPERATION_ID_1 = UUID.randomUUID(); private static final UUID OPERATION_ID_2 = UUID.randomUUID(); private static final UUID OPERATION_ID_3 = UUID.randomUUID(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java index 2edeaa999a1c7..046efc95fa5e9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WebBackendApiController.java @@ -9,6 +9,7 @@ import io.airbyte.api.model.generated.ConnectionStateType; import io.airbyte.api.model.generated.WebBackendCheckUpdatesRead; import io.airbyte.api.model.generated.WebBackendConnectionCreate; +import io.airbyte.api.model.generated.WebBackendConnectionListRequestBody; import io.airbyte.api.model.generated.WebBackendConnectionRead; import io.airbyte.api.model.generated.WebBackendConnectionReadList; import io.airbyte.api.model.generated.WebBackendConnectionRequestBody; @@ -16,7 +17,6 @@ import io.airbyte.api.model.generated.WebBackendGeographiesListResult; import io.airbyte.api.model.generated.WebBackendWorkspaceState; import io.airbyte.api.model.generated.WebBackendWorkspaceStateResult; -import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.server.handlers.WebBackendCheckUpdatesHandler; import io.airbyte.server.handlers.WebBackendConnectionsHandler; import io.airbyte.server.handlers.WebBackendGeographiesHandler; @@ -57,8 +57,8 @@ public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(final WebBacke } @Override - public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendListConnectionsForWorkspace(workspaceIdRequestBody)); + public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WebBackendConnectionListRequestBody webBackendConnectionListRequestBody) { + return ApiHelper.execute(() -> webBackendConnectionsHandler.webBackendListConnectionsForWorkspace(webBackendConnectionListRequestBody)); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java index a828448c77a9b..71f9e5dab36c6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java @@ -16,6 +16,7 @@ import io.airbyte.api.model.generated.DestinationRead; import io.airbyte.api.model.generated.DestinationReadList; import io.airbyte.api.model.generated.DestinationSearch; +import io.airbyte.api.model.generated.DestinationSnippetRead; import io.airbyte.api.model.generated.DestinationUpdate; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.json.Jsons; @@ -303,4 +304,14 @@ protected static DestinationRead toDestinationRead(final DestinationConnection d .icon(DestinationDefinitionsHandler.loadIcon(standardDestinationDefinition.getIcon())); } + protected static DestinationSnippetRead toDestinationSnippetRead(final DestinationConnection destinationConnection, + final StandardDestinationDefinition standardDestinationDefinition) { + return new DestinationSnippetRead() + .destinationId(destinationConnection.getDestinationId()) + .name(destinationConnection.getName()) + .destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) + .destinationName(standardDestinationDefinition.getName()) + .icon(DestinationDefinitionsHandler.loadIcon(standardDestinationDefinition.getIcon())); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java index ab1e1092d2359..6a24b5169c604 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java @@ -15,6 +15,7 @@ import io.airbyte.api.model.generated.SourceRead; import io.airbyte.api.model.generated.SourceReadList; import io.airbyte.api.model.generated.SourceSearch; +import io.airbyte.api.model.generated.SourceSnippetRead; import io.airbyte.api.model.generated.SourceUpdate; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.config.SourceConnection; @@ -319,4 +320,13 @@ protected static SourceRead toSourceRead(final SourceConnection sourceConnection .icon(SourceDefinitionsHandler.loadIcon(standardSourceDefinition.getIcon())); } + protected static SourceSnippetRead toSourceSnippetRead(final SourceConnection source, final StandardSourceDefinition sourceDefinition) { + return new SourceSnippetRead() + .sourceId(source.getSourceId()) + .name(source.getName()) + .sourceDefinitionId(sourceDefinition.getSourceDefinitionId()) + .sourceName(sourceDefinition.getName()) + .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index ea3aad8a66d33..362afe51c35b7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -20,6 +20,7 @@ import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.DestinationIdRequestBody; import io.airbyte.api.model.generated.DestinationRead; +import io.airbyte.api.model.generated.DestinationSnippetRead; import io.airbyte.api.model.generated.FieldTransform; import io.airbyte.api.model.generated.JobRead; import io.airbyte.api.model.generated.OperationCreate; @@ -31,11 +32,13 @@ import io.airbyte.api.model.generated.SourceDiscoverSchemaRequestBody; import io.airbyte.api.model.generated.SourceIdRequestBody; import io.airbyte.api.model.generated.SourceRead; +import io.airbyte.api.model.generated.SourceSnippetRead; import io.airbyte.api.model.generated.StreamDescriptor; import io.airbyte.api.model.generated.StreamTransform; import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum; import io.airbyte.api.model.generated.WebBackendConnectionCreate; import io.airbyte.api.model.generated.WebBackendConnectionListItem; +import io.airbyte.api.model.generated.WebBackendConnectionListRequestBody; import io.airbyte.api.model.generated.WebBackendConnectionRead; import io.airbyte.api.model.generated.WebBackendConnectionReadList; import io.airbyte.api.model.generated.WebBackendConnectionRequestBody; @@ -43,7 +46,6 @@ import io.airbyte.api.model.generated.WebBackendOperationCreateOrUpdate; import io.airbyte.api.model.generated.WebBackendWorkspaceState; import io.airbyte.api.model.generated.WebBackendWorkspaceStateResult; -import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.MoreBooleans; @@ -52,6 +54,7 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.handlers.helpers.CatalogConverter; @@ -106,23 +109,30 @@ public ConnectionStateType getStateType(final ConnectionIdRequestBody connection return Enums.convertTo(stateHandler.getState(connectionIdRequestBody).getStateType(), ConnectionStateType.class); } - public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) - throws IOException, JsonValidationException, ConfigNotFoundException { - - // passing 'false' so that deleted connections are not included - final List standardSyncs = - configRepository.listWorkspaceStandardSyncs(workspaceIdRequestBody.getWorkspaceId(), false); - final Map sourceReadById = - getSourceReadById(standardSyncs.stream().map(StandardSync::getSourceId).toList()); - final Map destinationReadById = - getDestinationReadById(standardSyncs.stream().map(StandardSync::getDestinationId).toList()); - final Map latestJobByConnectionId = - getLatestJobByConnectionId(standardSyncs.stream().map(StandardSync::getConnectionId).toList()); - final Map runningJobByConnectionId = - getRunningJobByConnectionId(standardSyncs.stream().map(StandardSync::getConnectionId).toList()); - + public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WebBackendConnectionListRequestBody webBackendConnectionListRequestBody) + throws IOException { + + final StandardSyncQuery query = new StandardSyncQuery( + webBackendConnectionListRequestBody.getWorkspaceId(), + webBackendConnectionListRequestBody.getSourceId(), + webBackendConnectionListRequestBody.getDestinationId(), + // passing 'false' so that deleted connections are not included + false); + + final List standardSyncs = configRepository.listWorkspaceStandardSyncs(query); + final List sourceIds = standardSyncs.stream().map(StandardSync::getSourceId).toList(); + final List destinationIds = standardSyncs.stream().map(StandardSync::getDestinationId).toList(); + final List connectionIds = standardSyncs.stream().map(StandardSync::getConnectionId).toList(); + + // Fetching all the related objects we need for the final output + final Map sourceReadById = getSourceSnippetReadById(sourceIds); + final Map destinationReadById = getDestinationSnippetReadById(destinationIds); + final Map latestJobByConnectionId = getLatestJobByConnectionId(connectionIds); + // This call could be removed, running jobs should be a subset of latest jobs, need to expose the + // right status filtering for this. + final Map runningJobByConnectionId = getRunningJobByConnectionId(connectionIds); final Map newestFetchEventsByActorId = - configRepository.getMostRecentActorCatalogFetchEventForSources(new ArrayList<>()); + configRepository.getMostRecentActorCatalogFetchEventForSources(sourceIds); final List connectionItems = Lists.newArrayList(); @@ -150,23 +160,19 @@ private Map getRunningJobByConnectionId(final List connecti .collect(Collectors.toMap(j -> UUID.fromString(j.getConfigId()), Function.identity())); } - private Map getSourceReadById(final List sourceIds) throws IOException { - final List sourceReads = configRepository.getSourceAndDefinitionsFromSourceIds(sourceIds) + private Map getSourceSnippetReadById(final List sourceIds) throws IOException { + return configRepository.getSourceAndDefinitionsFromSourceIds(sourceIds) .stream() - .map(sourceAndDefinition -> SourceHandler.toSourceRead(sourceAndDefinition.source(), sourceAndDefinition.definition())) - .toList(); - - return sourceReads.stream().collect(Collectors.toMap(SourceRead::getSourceId, Function.identity())); + .map(sourceAndDefinition -> SourceHandler.toSourceSnippetRead(sourceAndDefinition.source(), sourceAndDefinition.definition())) + .collect(Collectors.toMap(SourceSnippetRead::getSourceId, Function.identity())); } - private Map getDestinationReadById(final List destinationIds) throws IOException { - final List destinationReads = configRepository.getDestinationAndDefinitionsFromDestinationIds(destinationIds) + private Map getDestinationSnippetReadById(final List destinationIds) throws IOException { + return configRepository.getDestinationAndDefinitionsFromDestinationIds(destinationIds) .stream() - .map(destinationAndDefinition -> DestinationHandler.toDestinationRead(destinationAndDefinition.destination(), + .map(destinationAndDefinition -> DestinationHandler.toDestinationSnippetRead(destinationAndDefinition.destination(), destinationAndDefinition.definition())) - .toList(); - - return destinationReads.stream().collect(Collectors.toMap(DestinationRead::getDestinationId, Function.identity())); + .collect(Collectors.toMap(DestinationSnippetRead::getDestinationId, Function.identity())); } private WebBackendConnectionRead buildWebBackendConnectionRead(final ConnectionRead connectionRead, final Optional currentSourceCatalogId) @@ -197,38 +203,34 @@ private WebBackendConnectionRead buildWebBackendConnectionRead(final ConnectionR return webBackendConnectionRead; } - private WebBackendConnectionListItem buildWebBackendConnectionListItem( - final StandardSync standardSync, - final Map sourceReadById, - final Map destinationReadById, - final Map latestJobByConnectionId, - final Map runningJobByConnectionId, - final Optional latestFetchEvent) - throws JsonValidationException, ConfigNotFoundException, IOException { + static private WebBackendConnectionListItem buildWebBackendConnectionListItem( + final StandardSync standardSync, + final Map sourceReadById, + final Map destinationReadById, + final Map latestJobByConnectionId, + final Map runningJobByConnectionId, + final Optional latestFetchEvent) { - final SourceRead source = sourceReadById.get(standardSync.getSourceId()); - final DestinationRead destination = destinationReadById.get(standardSync.getDestinationId()); + final SourceSnippetRead source = sourceReadById.get(standardSync.getSourceId()); + final DestinationSnippetRead destination = destinationReadById.get(standardSync.getDestinationId()); final Optional latestSyncJob = Optional.ofNullable(latestJobByConnectionId.get(standardSync.getConnectionId())); final Optional latestRunningSyncJob = Optional.ofNullable(runningJobByConnectionId.get(standardSync.getConnectionId())); - final ConnectionRead connectionRead = connectionsHandler.getConnection(standardSync.getConnectionId()); + final ConnectionRead connectionRead = ApiPojoConverters.internalToConnectionRead(standardSync); final Optional currentCatalogId = connectionRead == null ? Optional.empty() : Optional.ofNullable(connectionRead.getSourceCatalogId()); final SchemaChange schemaChange = getSchemaChange(connectionRead, currentCatalogId, latestFetchEvent); final WebBackendConnectionListItem listItem = new WebBackendConnectionListItem() .connectionId(standardSync.getConnectionId()) - .sourceId(standardSync.getSourceId()) - .destinationId(standardSync.getDestinationId()) .status(ApiPojoConverters.toApiStatus(standardSync.getStatus())) .name(standardSync.getName()) .scheduleType(ApiPojoConverters.toApiConnectionScheduleType(standardSync)) .scheduleData(ApiPojoConverters.toApiConnectionScheduleData(standardSync)) .source(source) .destination(destination) + .isSyncing(latestRunningSyncJob.isPresent()) .schemaChange(schemaChange); - listItem.setIsSyncing(latestRunningSyncJob.isPresent()); - latestSyncJob.ifPresent(job -> { listItem.setLatestSyncJobCreatedAt(job.getCreatedAt()); listItem.setLatestSyncJobStatus(job.getStatus()); @@ -246,10 +248,10 @@ private WebBackendConnectionListItem buildWebBackendConnectionListItem( * existing actor catalog, there is a schema change. */ @VisibleForTesting - SchemaChange getSchemaChange( - final ConnectionRead connectionRead, - final Optional currentSourceCatalogId, - final Optional mostRecentFetchEvent) { + static SchemaChange getSchemaChange( + final ConnectionRead connectionRead, + final Optional currentSourceCatalogId, + final Optional mostRecentFetchEvent) { if (connectionRead == null || currentSourceCatalogId.isEmpty()) { return SchemaChange.NO_CHANGE; } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index b20c3b1fe4bf0..c81bfb97597c7 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -66,13 +66,13 @@ import io.airbyte.api.model.generated.SynchronousJobRead; import io.airbyte.api.model.generated.WebBackendConnectionCreate; import io.airbyte.api.model.generated.WebBackendConnectionListItem; +import io.airbyte.api.model.generated.WebBackendConnectionListRequestBody; import io.airbyte.api.model.generated.WebBackendConnectionRead; import io.airbyte.api.model.generated.WebBackendConnectionReadList; import io.airbyte.api.model.generated.WebBackendConnectionRequestBody; import io.airbyte.api.model.generated.WebBackendConnectionUpdate; import io.airbyte.api.model.generated.WebBackendOperationCreateOrUpdate; import io.airbyte.api.model.generated.WebBackendWorkspaceState; -import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; @@ -88,6 +88,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.ConfigRepository.DestinationAndDefinition; import io.airbyte.config.persistence.ConfigRepository.SourceAndDefinition; +import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; @@ -186,7 +187,7 @@ void setup() throws IOException, JsonValidationException, ConfigNotFoundExceptio final StandardSync brokenStandardSync = ConnectionHelpers.generateSyncWithSourceAndDestinationId(source.getSourceId(), destination.getDestinationId(), true, Status.INACTIVE); - when(configRepository.listWorkspaceStandardSyncs(sourceRead.getWorkspaceId(), false)) + when(configRepository.listWorkspaceStandardSyncs(new StandardSyncQuery(sourceRead.getWorkspaceId(), null, null, false))) .thenReturn(Collections.singletonList(standardSync)); when(configRepository.getSourceAndDefinitionsFromSourceIds(Collections.singletonList(source.getSourceId()))) .thenReturn(Collections.singletonList(new SourceAndDefinition(source, sourceDefinition))); @@ -366,11 +367,11 @@ void testGetWorkspaceStateEmpty() throws IOException { @Test void testWebBackendListConnectionsForWorkspace() throws IOException, JsonValidationException, ConfigNotFoundException { - final WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody(); - workspaceIdRequestBody.setWorkspaceId(sourceRead.getWorkspaceId()); + final WebBackendConnectionListRequestBody webBackendConnectionListRequestBody = new WebBackendConnectionListRequestBody(); + webBackendConnectionListRequestBody.setWorkspaceId(sourceRead.getWorkspaceId()); final WebBackendConnectionReadList WebBackendConnectionReadList = - wbHandler.webBackendListConnectionsForWorkspace(workspaceIdRequestBody); + wbHandler.webBackendListConnectionsForWorkspace(webBackendConnectionListRequestBody); assertEquals(1, WebBackendConnectionReadList.getConnections().size()); assertEquals(expectedListItem, WebBackendConnectionReadList.getConnections().get(0)); diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java index 223135d04669c..00afd4ff6a803 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java @@ -20,11 +20,13 @@ import io.airbyte.api.model.generated.ConnectionScheduleType; import io.airbyte.api.model.generated.ConnectionStatus; import io.airbyte.api.model.generated.DestinationRead; +import io.airbyte.api.model.generated.DestinationSnippetRead; import io.airbyte.api.model.generated.Geography; import io.airbyte.api.model.generated.JobStatus; import io.airbyte.api.model.generated.ResourceRequirements; import io.airbyte.api.model.generated.SchemaChange; import io.airbyte.api.model.generated.SourceRead; +import io.airbyte.api.model.generated.SourceSnippetRead; import io.airbyte.api.model.generated.SyncMode; import io.airbyte.api.model.generated.WebBackendConnectionListItem; import io.airbyte.commons.enums.Enums; @@ -264,10 +266,18 @@ public static WebBackendConnectionListItem generateExpectedWebBackendConnectionL final WebBackendConnectionListItem connectionListItem = new WebBackendConnectionListItem() .connectionId(standardSync.getConnectionId()) .name(standardSync.getName()) - .sourceId(standardSync.getSourceId()) - .destinationId(standardSync.getDestinationId()) - .source(source) - .destination(destination) + .source(new SourceSnippetRead() + .icon(source.getIcon()) + .name(source.getName()) + .sourceName(source.getSourceName()) + .sourceDefinitionId(source.getSourceDefinitionId()) + .sourceId(source.getSourceId())) + .destination(new DestinationSnippetRead() + .icon(destination.getIcon()) + .name(destination.getName()) + .destinationName(destination.getDestinationName()) + .destinationDefinitionId(destination.getDestinationDefinitionId()) + .destinationId(destination.getDestinationId())) .status(ApiPojoConverters.toApiStatus(standardSync.getStatus())) .isSyncing(isSyncing) .latestSyncJobCreatedAt(latestSyncJobCreatedAt) diff --git a/airbyte-webapp/src/components/EntityTable/utils.tsx b/airbyte-webapp/src/components/EntityTable/utils.tsx index 71df00f71a0a7..fa9c18b81acb7 100644 --- a/airbyte-webapp/src/components/EntityTable/utils.tsx +++ b/airbyte-webapp/src/components/EntityTable/utils.tsx @@ -1,40 +1,35 @@ import { ConnectionStatus, - DestinationDefinitionRead, DestinationRead, + DestinationSnippetRead, JobStatus, - SourceDefinitionRead, SourceRead, + SourceSnippetRead, WebBackendConnectionListItem, } from "core/request/AirbyteClient"; import { EntityTableDataItem, ITableDataItem, Status as ConnectionSyncStatus } from "./types"; -const getConnectorTypeName = (connectorSpec: DestinationRead | SourceRead) => { +const getConnectorTypeName = (connectorSpec: DestinationSnippetRead | SourceSnippetRead) => { return "sourceName" in connectorSpec ? connectorSpec.sourceName : connectorSpec.destinationName; }; +const getConnectorTypeId = (connectorSpec: DestinationSnippetRead | SourceSnippetRead) => { + return "sourceId" in connectorSpec ? connectorSpec.sourceId : connectorSpec.destinationId; +}; + // TODO: types in next methods look a bit ugly export function getEntityTableData< S extends "source" | "destination", - SoD extends S extends "source" ? SourceRead : DestinationRead, - Def extends S extends "source" ? SourceDefinitionRead : DestinationDefinitionRead ->(entities: SoD[], connections: WebBackendConnectionListItem[], definitions: Def[], type: S): EntityTableDataItem[] { + SoD extends S extends "source" ? SourceRead : DestinationRead +>(entities: SoD[], connections: WebBackendConnectionListItem[], type: S): EntityTableDataItem[] { const connectType = type === "source" ? "destination" : "source"; const mappedEntities = entities.map((entityItem) => { const entitySoDId = entityItem[`${type}Id` as keyof SoD] as unknown as string; const entitySoDName = entityItem[`${type}Name` as keyof SoD] as unknown as string; const entityConnections = connections.filter( - (connectionItem) => connectionItem[`${type}Id` as "sourceId" | "destinationId"] === entitySoDId - ); - - const definitionId = `${type}DefinitionId` as keyof Def; - const entityDefinitionId = entityItem[`${type}DefinitionId` as keyof SoD]; - - const definition = definitions.find( - // @ts-expect-error ignored during react-scripts update - (def) => def[definitionId] === entityDefinitionId + (connectionItem) => getConnectorTypeId(connectionItem[type]) === entitySoDId ); if (!entityConnections.length) { @@ -43,7 +38,7 @@ export function getEntityTableData< entityName: entityItem.name, enabled: true, connectorName: entitySoDName, - connectorIcon: definition?.icon, + connectorIcon: entityItem.icon, lastSync: null, connectEntities: [], }; @@ -69,7 +64,7 @@ export function getEntityTableData< connectorName: entitySoDName, lastSync: sortBySync?.[0].latestSyncJobCreatedAt, connectEntities, - connectorIcon: definition?.icon, + connectorIcon: entityItem.icon, }; }); diff --git a/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx b/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx index 64ebc0cba9f5a..38af6be321022 100644 --- a/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx +++ b/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx @@ -7,7 +7,6 @@ import { getEntityTableData } from "components/EntityTable/utils"; import { DestinationRead } from "core/request/AirbyteClient"; import { useConnectionList } from "hooks/services/useConnectionHook"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; interface DestinationsTableProps { destinations: DestinationRead[]; @@ -16,9 +15,8 @@ interface DestinationsTableProps { export const DestinationsTable: React.FC = ({ destinations }) => { const navigate = useNavigate(); const { connections } = useConnectionList(); - const { destinationDefinitions } = useDestinationDefinitionList(); - const data = getEntityTableData(destinations, connections, destinationDefinitions, "destination"); + const data = getEntityTableData(destinations, connections, "destination"); const clickRow = (destination: EntityTableDataItem) => navigate(`${destination.entityId}`); diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/SourcesTable.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/SourcesTable.tsx index dc8e53bfea831..f6b028b7833c6 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/SourcesTable.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/SourcesTable.tsx @@ -8,8 +8,6 @@ import { getEntityTableData } from "components/EntityTable/utils"; import { SourceRead } from "core/request/AirbyteClient"; import { useConnectionList } from "hooks/services/useConnectionHook"; -import { useSourceDefinitionList } from "../../../../../services/connector/SourceDefinitionService"; - interface SourcesTableProps { sources: SourceRead[]; } @@ -18,9 +16,8 @@ const SourcesTable: React.FC = ({ sources }) => { const navigate = useNavigate(); const { connections } = useConnectionList(); - const { sourceDefinitions } = useSourceDefinitionList(); - const data = getEntityTableData(sources, connections, sourceDefinitions, "source"); + const data = getEntityTableData(sources, connections, "source"); const clickRow = (source: EntityTableDataItem) => navigate(`${source.entityId}`); diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx index 8f67a78c17e94..ef0ccd18a6a6a 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx @@ -52,7 +52,7 @@ const SourceItemPage: React.FC = () => { { label: source.name }, ]; - const connectionsWithSource = connections.filter((connectionItem) => connectionItem.sourceId === source.sourceId); + const connectionsWithSource = connections.filter(({ source: { sourceId } }) => sourceId === source.sourceId); const destinationDropdownOptions: DropdownMenuOptionType[] = useMemo( () => diff --git a/airbyte-webapp/src/pages/destination/DestinationOverviewPage.tsx b/airbyte-webapp/src/pages/destination/DestinationOverviewPage.tsx index 93115750a16f6..b5e8d9d8576cb 100644 --- a/airbyte-webapp/src/pages/destination/DestinationOverviewPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationOverviewPage.tsx @@ -25,7 +25,7 @@ export const DestinationOverviewPage = () => { const { sourceDefinitions } = useSourceDefinitionList(); const connectionsWithDestination = connections.filter( - (connectionItem) => connectionItem.destinationId === destination.destinationId + ({ destination: { destinationId } }) => destinationId === destination.destinationId ); const sourceDropdownOptions: DropdownMenuOptionType[] = useMemo( diff --git a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx index 2c6842578f9b8..37532cae4e0d3 100644 --- a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx @@ -21,7 +21,7 @@ export const DestinationSettingsPage: React.FC = () => { const destination = useGetDestination(params.id); const { connections } = useConnectionList(); const connectionsWithDestination = connections.filter( - (connectionItem) => connectionItem.destinationId === destination.destinationId + ({ destination: { destinationId } }) => destinationId === destination.destinationId ); const destinationSpecification = useGetDestinationDefinitionSpecification(destination.destinationDefinitionId); const destinationDefinition = useDestinationDefinition(destination.destinationDefinitionId); diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 1dec3f64555b2..cef598e12e04b 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -8866,7 +8866,7 @@

Consumes

Request body

-
WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
+
WebBackendConnectionListRequestBody WebBackendConnectionListRequestBody (required)
Body Parameter
@@ -8887,34 +8887,24 @@

Example data

Content-Type: application/json
{
   "connections" : [ {
-    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "latestSyncJobCreatedAt" : 0,
+    "name" : "name",
     "destination" : {
-      "connectionConfiguration" : {
-        "user" : "charles"
-      },
       "destinationName" : "destinationName",
       "name" : "name",
       "icon" : "icon",
       "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     },
+    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "isSyncing" : true,
     "source" : {
       "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "connectionConfiguration" : {
-        "user" : "charles"
-      },
       "name" : "name",
       "icon" : "icon",
       "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "sourceName" : "sourceName",
-      "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      "sourceName" : "sourceName"
     },
-    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "name" : "name",
-    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "scheduleData" : {
       "cron" : {
         "cronExpression" : "cronExpression",
@@ -8926,34 +8916,24 @@ 

Example data

} } }, { - "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "latestSyncJobCreatedAt" : 0, + "name" : "name", "destination" : { - "connectionConfiguration" : { - "user" : "charles" - }, "destinationName" : "destinationName", "name" : "name", "icon" : "icon", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", - "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", - "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" + "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, + "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "isSyncing" : true, "source" : { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", - "connectionConfiguration" : { - "user" : "charles" - }, "name" : "name", "icon" : "icon", "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", - "sourceName" : "sourceName", - "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" + "sourceName" : "sourceName" }, - "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", - "name" : "name", - "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", "scheduleData" : { "cron" : { "cronExpression" : "cronExpression", @@ -10098,6 +10078,7 @@

Table of Contents

  • DestinationRead -
  • DestinationReadList -
  • DestinationSearch -
  • +
  • DestinationSnippetRead -
  • DestinationSyncMode -
  • DestinationUpdate -
  • FieldAdd -
  • @@ -10183,6 +10164,7 @@

    Table of Contents

  • SourceRead -
  • SourceReadList -
  • SourceSearch -
  • +
  • SourceSnippetRead -
  • SourceUpdate -
  • StreamDescriptor -
  • StreamState -
  • @@ -10193,6 +10175,7 @@

    Table of Contents

  • WebBackendCheckUpdatesRead -
  • WebBackendConnectionCreate -
  • WebBackendConnectionListItem -
  • +
  • WebBackendConnectionListRequestBody -
  • WebBackendConnectionRead -
  • WebBackendConnectionReadList -
  • WebBackendConnectionRequestBody -
  • @@ -10838,6 +10821,17 @@

    DestinationSearch - destinationName (optional)

    +
    +

    DestinationSnippetRead - Up

    +
    +
    +
    destinationId
    UUID format: uuid
    +
    name
    +
    destinationDefinitionId
    UUID format: uuid
    +
    destinationName
    +
    icon (optional)
    +
    +
    +
    +

    SourceSnippetRead - Up

    +
    +
    +
    sourceId
    UUID format: uuid
    +
    name
    +
    sourceDefinitionId
    UUID format: uuid
    +
    sourceName
    +
    icon (optional)
    +
    +

    SourceUpdate - Up

    @@ -11677,19 +11682,26 @@

    WebBackendConnectionListItem
    connectionId
    UUID format: uuid
    name
    -
    sourceId
    UUID format: uuid
    -
    destinationId
    UUID format: uuid
    scheduleType (optional)
    scheduleData (optional)
    status
    -
    source
    -
    destination
    +
    source
    +
    destination
    latestSyncJobCreatedAt (optional)
    Long epoch time of the latest sync job. null if no sync job has taken place. format: int64
    latestSyncJobStatus (optional)
    isSyncing
    schemaChange

    +
    +

    WebBackendConnectionListRequestBody - Up

    +
    +
    +
    workspaceId
    UUID format: uuid
    +
    sourceId (optional)
    UUID format: uuid
    +
    destinationId (optional)
    UUID format: uuid
    +
    +