Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

document that web backends should not have db access #22509

Merged
merged 1 commit into from
Feb 7, 2023
Merged

document that web backends should not have db access #22509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions ...erver/src/main/java/io/airbyte/commons/server/handlers/WebBackendCheckUpdatesHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,24 @@
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;

/**
* The web backend is an abstraction that allows the frontend to structure data in such a way that
* it is easier for a react frontend to consume. It should NOT have direct access to the database.
* It should operate exclusively by calling other endpoints that are exposed in the API.
**/
@Slf4j
@Singleton
public class WebBackendCheckUpdatesHandler {

private static final int NO_CHANGES_FOUND = 0;

final ConfigRepository configRepository;
// todo (cgardens) - this handler should NOT have access to the db. only access via handler.
@Deprecated
final ConfigRepository configRepositoryDoNotUse;
final AirbyteGithubStore githubStore;

public WebBackendCheckUpdatesHandler(final ConfigRepository configRepository, final AirbyteGithubStore githubStore) {
this.configRepository = configRepository;
public WebBackendCheckUpdatesHandler(final ConfigRepository configRepositoryDoNotUse, final AirbyteGithubStore githubStore) {
this.configRepositoryDoNotUse = configRepositoryDoNotUse;
this.githubStore = githubStore;
}

Expand All @@ -47,7 +54,7 @@ private int getDestinationDiffCount() {
final Map<UUID, String> newActorDefToDockerImageTag;

try {
currentActorDefToDockerImageTag = configRepository.listStandardDestinationDefinitions(false)
currentActorDefToDockerImageTag = configRepositoryDoNotUse.listStandardDestinationDefinitions(false)
.stream()
.map(def -> Map.entry(def.getDestinationDefinitionId(), def.getDockerImageTag()))
.toList();
Expand All @@ -73,7 +80,7 @@ private int getSourceDiffCount() {
final Map<UUID, String> newActorDefToDockerImageTag;

try {
currentActorDefToDockerImageTag = configRepository.listStandardSourceDefinitions(false)
currentActorDefToDockerImageTag = configRepositoryDoNotUse.listStandardSourceDefinitions(false)
.stream()
.map(def -> Map.entry(def.getSourceDefinitionId(), def.getDockerImageTag()))
.toList();
Expand Down
42 changes: 25 additions & 17 deletions ...server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@
import java.util.function.Function;
import java.util.stream.Collectors;

/**
* The web backend is an abstraction that allows the frontend to structure data in such a way that
* it is easier for a react frontend to consume. It should NOT have direct access to the database.
* It should operate exclusively by calling other endpoints that are exposed in the API.
**/
@Singleton
public class WebBackendConnectionsHandler {

Expand All @@ -87,7 +92,8 @@ public class WebBackendConnectionsHandler {
private final OperationsHandler operationsHandler;
private final EventRunner eventRunner;
// todo (cgardens) - this handler should NOT have access to the db. only access via handler.
private final ConfigRepository configRepository;
@Deprecated
private final ConfigRepository configRepositoryDoNotUse;

public WebBackendConnectionsHandler(final ConnectionsHandler connectionsHandler,
final StateHandler stateHandler,
Expand All @@ -97,7 +103,7 @@ public WebBackendConnectionsHandler(final ConnectionsHandler connectionsHandler,
final SchedulerHandler schedulerHandler,
final OperationsHandler operationsHandler,
final EventRunner eventRunner,
final ConfigRepository configRepository) {
final ConfigRepository configRepositoryDoNotUse) {
this.connectionsHandler = connectionsHandler;
this.stateHandler = stateHandler;
this.sourceHandler = sourceHandler;
Expand All @@ -106,14 +112,14 @@ public WebBackendConnectionsHandler(final ConnectionsHandler connectionsHandler,
this.schedulerHandler = schedulerHandler;
this.operationsHandler = operationsHandler;
this.eventRunner = eventRunner;
this.configRepository = configRepository;
this.configRepositoryDoNotUse = configRepositoryDoNotUse;
}

public WebBackendWorkspaceStateResult getWorkspaceState(final WebBackendWorkspaceState webBackendWorkspaceState) throws IOException {
final var workspaceId = webBackendWorkspaceState.getWorkspaceId();
final var connectionCount = configRepository.countConnectionsForWorkspace(workspaceId);
final var destinationCount = configRepository.countDestinationsForWorkspace(workspaceId);
final var sourceCount = configRepository.countSourcesForWorkspace(workspaceId);
final var connectionCount = configRepositoryDoNotUse.countConnectionsForWorkspace(workspaceId);
final var destinationCount = configRepositoryDoNotUse.countDestinationsForWorkspace(workspaceId);
final var sourceCount = configRepositoryDoNotUse.countSourcesForWorkspace(workspaceId);

return new WebBackendWorkspaceStateResult()
.hasConnections(connectionCount > 0)
Expand All @@ -135,7 +141,7 @@ public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final
// passing 'false' so that deleted connections are not included
false);

final List<StandardSync> standardSyncs = configRepository.listWorkspaceStandardSyncs(query);
final List<StandardSync> standardSyncs = configRepositoryDoNotUse.listWorkspaceStandardSyncs(query);
final List<UUID> sourceIds = standardSyncs.stream().map(StandardSync::getSourceId).toList();
final List<UUID> destinationIds = standardSyncs.stream().map(StandardSync::getDestinationId).toList();
final List<UUID> connectionIds = standardSyncs.stream().map(StandardSync::getConnectionId).toList();
Expand All @@ -148,7 +154,7 @@ public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final
// right status filtering for this.
final Map<UUID, JobRead> runningJobByConnectionId = getRunningJobByConnectionId(connectionIds);
final Map<UUID, ActorCatalogFetchEvent> newestFetchEventsByActorId =
configRepository.getMostRecentActorCatalogFetchEventForSources(sourceIds);
configRepositoryDoNotUse.getMostRecentActorCatalogFetchEventForSources(sourceIds);

final List<WebBackendConnectionListItem> connectionItems = Lists.newArrayList();

Expand Down Expand Up @@ -177,14 +183,14 @@ private Map<UUID, JobRead> getRunningJobByConnectionId(final List<UUID> connecti
}

private Map<UUID, SourceSnippetRead> getSourceSnippetReadById(final List<UUID> sourceIds) throws IOException {
return configRepository.getSourceAndDefinitionsFromSourceIds(sourceIds)
return configRepositoryDoNotUse.getSourceAndDefinitionsFromSourceIds(sourceIds)
.stream()
.map(sourceAndDefinition -> SourceHandler.toSourceSnippetRead(sourceAndDefinition.source(), sourceAndDefinition.definition()))
.collect(Collectors.toMap(SourceSnippetRead::getSourceId, Function.identity()));
}

private Map<UUID, DestinationSnippetRead> getDestinationSnippetReadById(final List<UUID> destinationIds) throws IOException {
return configRepository.getDestinationAndDefinitionsFromDestinationIds(destinationIds)
return configRepositoryDoNotUse.getDestinationAndDefinitionsFromDestinationIds(destinationIds)
.stream()
.map(destinationAndDefinition -> DestinationHandler.toDestinationSnippetRead(destinationAndDefinition.destination(),
destinationAndDefinition.definition()))
Expand All @@ -210,7 +216,7 @@ private WebBackendConnectionRead buildWebBackendConnectionRead(final ConnectionR
});

final Optional<ActorCatalogFetchEvent> mostRecentFetchEvent =
configRepository.getMostRecentActorCatalogFetchEventForSource(connectionRead.getSourceId());
configRepositoryDoNotUse.getMostRecentActorCatalogFetchEventForSource(connectionRead.getSourceId());

final SchemaChange schemaChange = getSchemaChange(connectionRead, currentSourceCatalogId, mostRecentFetchEvent);

Expand Down Expand Up @@ -539,13 +545,15 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne
if (webBackendConnectionPatch.getSyncCatalog() != null) {
// Get the most recent actor catalog fetched for this connection's source and the newly updated sync
// catalog
Optional<ActorCatalog> mostRecentActorCatalog = configRepository.getMostRecentActorCatalogForSource(originalConnectionRead.getSourceId());
AirbyteCatalog newAirbyteCatalog = webBackendConnectionPatch.getSyncCatalog();
final Optional<ActorCatalog> mostRecentActorCatalog =
configRepositoryDoNotUse.getMostRecentActorCatalogForSource(originalConnectionRead.getSourceId());
final AirbyteCatalog newAirbyteCatalog = webBackendConnectionPatch.getSyncCatalog();
// Get the diff between these two catalogs to check for breaking changes
if (mostRecentActorCatalog.isPresent()) {
final io.airbyte.protocol.models.AirbyteCatalog mostRecentAirbyteCatalog =
Jsons.object(mostRecentActorCatalog.get().getCatalog(), io.airbyte.protocol.models.AirbyteCatalog.class);
final StandardSourceDefinition sourceDefinition = configRepository.getSourceDefinitionFromSource(originalConnectionRead.getSourceId());
final StandardSourceDefinition sourceDefinition =
configRepositoryDoNotUse.getSourceDefinitionFromSource(originalConnectionRead.getSourceId());
final CatalogDiff catalogDiff =
connectionsHandler.getDiff(newAirbyteCatalog, CatalogConverter.toApi(mostRecentAirbyteCatalog, sourceDefinition),
CatalogConverter.toConfiguredProtocol(newAirbyteCatalog));
Expand All @@ -556,7 +564,7 @@ public WebBackendConnectionRead webBackendUpdateConnection(final WebBackendConne
// before doing any updates, fetch the existing catalog so that it can be diffed
// with the final catalog to determine which streams might need to be reset.
final ConfiguredAirbyteCatalog oldConfiguredCatalog =
configRepository.getConfiguredCatalogForConnection(connectionId);
configRepositoryDoNotUse.getConfiguredCatalogForConnection(connectionId);

final List<UUID> newAndExistingOperationIds = createOrUpdateOperations(originalConnectionRead, webBackendConnectionPatch);

Expand Down Expand Up @@ -619,7 +627,7 @@ private void resetStreamsIfNeeded(final WebBackendConnectionUpdate webBackendCon
final ConnectionStateType stateType = getStateType(connectionIdRequestBody);

if (stateType == ConnectionStateType.LEGACY || stateType == ConnectionStateType.NOT_SET) {
streamsToReset = configRepository.getAllStreamsForConnection(connectionId);
streamsToReset = configRepositoryDoNotUse.getAllStreamsForConnection(connectionId);
}
eventRunner.resetConnection(
connectionId,
Expand Down Expand Up @@ -728,7 +736,7 @@ protected static ConnectionCreate toConnectionCreate(final WebBackendConnectionC
@VisibleForTesting
protected static ConnectionUpdate toConnectionPatch(final WebBackendConnectionUpdate webBackendConnectionPatch,
final List<UUID> finalOperationIds,
boolean breakingChange) {
final boolean breakingChange) {
final ConnectionUpdate connectionPatch = new ConnectionUpdate();

connectionPatch.connectionId(webBackendConnectionPatch.getConnectionId());
Expand Down
5 changes: 5 additions & 0 deletions ...server/src/main/java/io/airbyte/commons/server/handlers/WebBackendGeographiesHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@
import java.util.Arrays;
import java.util.Collections;

/**
* The web backend is an abstraction that allows the frontend to structure data in such a way that
* it is easier for a react frontend to consume. It should NOT have direct access to the database.
* It should operate exclusively by calling other endpoints that are exposed in the API.
**/
@Singleton
public class WebBackendGeographiesHandler {

Expand Down