diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java index 39a87d72fac..99333df7b92 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java @@ -362,7 +362,8 @@ public Mono reconstructArtifactExchangeJsonFromFilesInRepo getApplicationResource(applicationReference.getMetadata(), ApplicationJson.class); ApplicationJson applicationJson = getApplicationJsonFromGitReference(applicationReference); copyNestedNonNullProperties(metadata, applicationJson); - return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson); + return jsonSchemaMigration.migrateApplicationJsonToLatestSchema( + applicationJson, baseArtifactId, branchName); }); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaMigration.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaMigration.java index 047c037e6c4..03d543dd3d0 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaMigration.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaMigration.java @@ -6,35 +6,27 @@ import com.appsmith.server.exceptions.AppsmithError; import com.appsmith.server.exceptions.AppsmithException; import com.appsmith.server.helpers.CollectionUtils; +import com.appsmith.server.migrations.utils.JsonSchemaMigrationHelper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; +import java.util.Map; + @Slf4j @Component @RequiredArgsConstructor public class JsonSchemaMigration { private final JsonSchemaVersions jsonSchemaVersions; + private final JsonSchemaMigrationHelper jsonSchemaMigrationHelper; private boolean isCompatible(ApplicationJson applicationJson) { return (applicationJson.getClientSchemaVersion() <= jsonSchemaVersions.getClientVersion()) && (applicationJson.getServerSchemaVersion() <= jsonSchemaVersions.getServerVersion()); } - /** - * This is a temporary check which is being placed for the compatibility of server versions in scenarios - * where user is moving a json from an instance which has - * release_autocommit_feature_enabled true to an instance which has the flag as false. In that case the server - * version number of json would be 8 and in new instance it would be not compatible. - * @param applicationJson - * @return - */ - private boolean isAutocommitVersionBump(ApplicationJson applicationJson) { - return jsonSchemaVersions.getServerVersion() == 7 && applicationJson.getServerSchemaVersion() == 8; - } - private void setSchemaVersions(ApplicationJson applicationJson) { applicationJson.setServerSchemaVersion(getCorrectSchemaVersion(applicationJson.getServerSchemaVersion())); applicationJson.setClientSchemaVersion(getCorrectSchemaVersion(applicationJson.getClientSchemaVersion())); @@ -53,24 +45,53 @@ public Mono migrateArtifactExchangeJsonToLatestS ArtifactExchangeJson artifactExchangeJson) { if (ArtifactType.APPLICATION.equals(artifactExchangeJson.getArtifactJsonType())) { - return migrateApplicationJsonToLatestSchema((ApplicationJson) artifactExchangeJson); + return migrateApplicationJsonToLatestSchema((ApplicationJson) artifactExchangeJson, null, null); } return Mono.fromCallable(() -> artifactExchangeJson); } - public Mono migrateApplicationJsonToLatestSchema(ApplicationJson applicationJson) { + public Mono migrateApplicationJsonToLatestSchema( + ApplicationJson applicationJson, String baseApplicationId, String branchName) { return Mono.fromCallable(() -> { setSchemaVersions(applicationJson); - if (isCompatible(applicationJson)) { - return migrateServerSchema(applicationJson); - } - - if (isAutocommitVersionBump(applicationJson)) { - return migrateServerSchema(applicationJson); + return applicationJson; + }) + .flatMap(appJson -> { + if (!isCompatible(appJson)) { + return Mono.empty(); } - return null; + // Taking a tech debt over here for import of file application. + // All migration above version 9 is reactive + // TODO: make import flow migration reactive + return Mono.just(migrateServerSchema(appJson)) + .flatMap(migratedApplicationJson -> { + if (migratedApplicationJson.getServerSchemaVersion() == 9 + && Boolean.TRUE.equals(MigrationHelperMethods.doesRestApiRequireMigration( + migratedApplicationJson))) { + return jsonSchemaMigrationHelper + .addDatasourceConfigurationToDefaultRestApiActions( + baseApplicationId, branchName, migratedApplicationJson) + .map(applicationJsonWithMigration10 -> { + applicationJsonWithMigration10.setServerSchemaVersion(10); + return applicationJsonWithMigration10; + }); + } + + migratedApplicationJson.setServerSchemaVersion(10); + return Mono.just(migratedApplicationJson); + }) + .map(migratedAppJson -> { + if (applicationJson + .getServerSchemaVersion() + .equals(jsonSchemaVersions.getServerVersion())) { + return applicationJson; + } + + applicationJson.setServerSchemaVersion(jsonSchemaVersions.getServerVersion()); + return applicationJson; + }); }) .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.INCOMPATIBLE_IMPORTED_JSON))); } @@ -81,7 +102,7 @@ public Mono migrateApplicationJsonToLatestSchema(ApplicationJso * @param artifactExchangeJson : the json to be imported * @return transformed artifact exchange json */ - @Deprecated + @Deprecated(forRemoval = true, since = "Use migrateArtifactJsonToLatestSchema") public ArtifactExchangeJson migrateArtifactToLatestSchema(ArtifactExchangeJson artifactExchangeJson) { if (!ArtifactType.APPLICATION.equals(artifactExchangeJson.getArtifactJsonType())) { @@ -91,11 +112,11 @@ public ArtifactExchangeJson migrateArtifactToLatestSchema(ArtifactExchangeJson a ApplicationJson applicationJson = (ApplicationJson) artifactExchangeJson; setSchemaVersions(applicationJson); if (!isCompatible(applicationJson)) { - if (!isAutocommitVersionBump(applicationJson)) { - throw new AppsmithException(AppsmithError.INCOMPATIBLE_IMPORTED_JSON); - } + throw new AppsmithException(AppsmithError.INCOMPATIBLE_IMPORTED_JSON); } - return migrateServerSchema(applicationJson); + + applicationJson = migrateServerSchema(applicationJson); + return nonReactiveServerMigrationForImport(applicationJson); } /** @@ -145,11 +166,37 @@ private ApplicationJson migrateServerSchema(ApplicationJson applicationJson) { MigrationHelperMethods.ensureXmlParserPresenceInCustomJsLibList(applicationJson); applicationJson.setServerSchemaVersion(7); case 7: + applicationJson.setServerSchemaVersion(8); case 8: MigrationHelperMethods.migrateThemeSettingsForAnvil(applicationJson); applicationJson.setServerSchemaVersion(9); + + // This is not supposed to have anymore additions to the schema. default: // Unable to detect the serverSchema + + } + + return applicationJson; + } + + /** + * This method is an alternative to reactive way of adding migrations to application json. + * this is getting used by flows which haven't implemented the reactive way yet. + * @param applicationJson : application json for which migration has to be done. + * @return return application json after migration + */ + private ApplicationJson nonReactiveServerMigrationForImport(ApplicationJson applicationJson) { + if (jsonSchemaVersions.getServerVersion().equals(applicationJson.getServerSchemaVersion())) { + return applicationJson; + } + + switch (applicationJson.getServerSchemaVersion()) { + case 9: + // this if for cases where we have empty datasource configs + MigrationHelperMethods.migrateApplicationJsonToVersionTen(applicationJson, Map.of()); + applicationJson.setServerSchemaVersion(10); + default: } if (applicationJson.getServerSchemaVersion().equals(jsonSchemaVersions.getServerVersion())) { diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaVersionsFallback.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaVersionsFallback.java index 63b6c7e7d4e..06518c18b4e 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaVersionsFallback.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaVersionsFallback.java @@ -4,7 +4,7 @@ @Component public class JsonSchemaVersionsFallback { - private static final Integer serverVersion = 9; + private static final Integer serverVersion = 10; public static final Integer clientVersion = 1; public Integer getServerVersion() { diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java index cf18759a867..87e3d93e6ff 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java @@ -1,8 +1,11 @@ package com.appsmith.server.migrations; +import com.appsmith.external.constants.PluginConstants; import com.appsmith.external.helpers.MustacheHelper; import com.appsmith.external.models.ActionDTO; import com.appsmith.external.models.BaseDomain; +import com.appsmith.external.models.Datasource; +import com.appsmith.external.models.DatasourceConfiguration; import com.appsmith.external.models.InvisibleActionFields; import com.appsmith.external.models.Property; import com.appsmith.server.constants.ApplicationConstants; @@ -1231,4 +1234,106 @@ public static void setThemeSettings(Application.ThemeSetting themeSetting) { themeSetting.setSizing(1); } } + + private static boolean conditionForDefaultRestDatasourceMigration(NewAction action) { + Datasource actionDatasource = action.getUnpublishedAction().getDatasource(); + + // condition to check if the action is default rest datasource. + // it has no datasource id and name is equal to DEFAULT_REST_DATASOURCE + boolean isActionDefaultRestDatasource = !org.springframework.util.StringUtils.hasText(actionDatasource.getId()) + && PluginConstants.DEFAULT_REST_DATASOURCE.equals(actionDatasource.getName()); + + // condition to check if the action has missing url or has no config at all + boolean isDatasourceConfigurationOrUrlMissing = actionDatasource.getDatasourceConfiguration() == null + || !org.springframework.util.StringUtils.hasText( + actionDatasource.getDatasourceConfiguration().getUrl()); + + return isActionDefaultRestDatasource && isDatasourceConfigurationOrUrlMissing; + } + + /** + * Adds datasource configuration and relevant url to the embedded datasource actions. + * @param applicationJson: ApplicationJson for which the migration has to be performed + * @param defaultDatasourceActionMap: gitSyncId to actions with default rest datasource map + */ + public static void migrateApplicationJsonToVersionTen( + ApplicationJson applicationJson, Map defaultDatasourceActionMap) { + List actionList = applicationJson.getActionList(); + if (CollectionUtils.isNullOrEmpty(actionList)) { + return; + } + + for (NewAction action : actionList) { + if (action.getUnpublishedAction() == null + || action.getUnpublishedAction().getDatasource() == null) { + continue; + } + + Datasource actionDatasource = action.getUnpublishedAction().getDatasource(); + if (conditionForDefaultRestDatasourceMigration(action)) { + // Idea is to add datasourceConfiguration to existing DEFAULT_REST_DATASOURCE apis, + // for which the datasource configuration is missing + // the url would be set to empty string as right url is not present over here. + setDatasourceConfigDetailsInDefaultRestDatasourceForActions(action, defaultDatasourceActionMap); + } + } + } + + /** + * Finds if the applicationJson has any default rest datasource which has a null datasource configuration + * or an unset url. + * @param applicationJson : Application Json for which requirement is to be checked. + * @return true if the application has a rest api which doesn't have a valid datasource configuration. + */ + public static Boolean doesRestApiRequireMigration(ApplicationJson applicationJson) { + List actionList = applicationJson.getActionList(); + if (CollectionUtils.isNullOrEmpty(actionList)) { + return Boolean.FALSE; + } + + for (NewAction action : actionList) { + if (action.getUnpublishedAction() == null + || action.getUnpublishedAction().getDatasource() == null) { + continue; + } + + Datasource actionDatasource = action.getUnpublishedAction().getDatasource(); + if (conditionForDefaultRestDatasourceMigration(action)) { + return Boolean.TRUE; + } + } + + return Boolean.FALSE; + } + + /** + * Adds the relevant url in the default rest datasource for the given action from an action in the db + * otherwise sets the url to empty + * it's established that action doesn't have the datasource. + * @param action : default rest datasource actions which doesn't have valid datasource configuration. + * @param defaultDatasourceActionMap : gitSyncId to actions with default rest datasource map + */ + public static void setDatasourceConfigDetailsInDefaultRestDatasourceForActions( + NewAction action, Map defaultDatasourceActionMap) { + + ActionDTO actionDTO = action.getUnpublishedAction(); + Datasource actionDatasource = actionDTO.getDatasource(); + DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration(); + + if (defaultDatasourceActionMap.containsKey(action.getGitSyncId())) { + NewAction actionFromMap = defaultDatasourceActionMap.get(action.getGitSyncId()); + DatasourceConfiguration datasourceConfigurationFromDBAction = + actionFromMap.getUnpublishedAction().getDatasource().getDatasourceConfiguration(); + + if (datasourceConfigurationFromDBAction != null) { + datasourceConfiguration.setUrl(datasourceConfigurationFromDBAction.getUrl()); + } + } + + if (!org.springframework.util.StringUtils.hasText(datasourceConfiguration.getUrl())) { + datasourceConfiguration.setUrl(""); + } + + actionDatasource.setDatasourceConfiguration(datasourceConfiguration); + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/utils/JsonSchemaMigrationHelper.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/utils/JsonSchemaMigrationHelper.java new file mode 100644 index 00000000000..934bf79c6ed --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/utils/JsonSchemaMigrationHelper.java @@ -0,0 +1,75 @@ +package com.appsmith.server.migrations.utils; + +import com.appsmith.external.constants.PluginConstants; +import com.appsmith.server.applications.base.ApplicationService; +import com.appsmith.server.domains.Application; +import com.appsmith.server.domains.NewAction; +import com.appsmith.server.dtos.ApplicationJson; +import com.appsmith.server.migrations.MigrationHelperMethods; +import com.appsmith.server.newactions.base.NewActionService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Mono; + +import java.util.Map; +import java.util.Optional; + +@Component +@Slf4j +@RequiredArgsConstructor +public class JsonSchemaMigrationHelper { + + private final ApplicationService applicationService; + private final NewActionService newActionService; + + public Mono addDatasourceConfigurationToDefaultRestApiActions( + String baseApplicationId, String branchName, ApplicationJson applicationJson) { + + Mono contingencyMigrationJson = Mono.defer(() -> Mono.fromCallable(() -> { + MigrationHelperMethods.migrateApplicationJsonToVersionTen(applicationJson, Map.of()); + return applicationJson; + })); + + if (!StringUtils.hasText(baseApplicationId) || !StringUtils.hasText(branchName)) { + return contingencyMigrationJson; + } + + Mono applicationMono = applicationService + .findByBranchNameAndBaseApplicationId(branchName, baseApplicationId, null) + .cache(); + + return applicationMono + .flatMap(branchedApplication -> { + return newActionService + .findAllByApplicationIdAndViewMode( + branchedApplication.getId(), Boolean.FALSE, Optional.empty(), Optional.empty()) + .filter(action -> { + if (action.getUnpublishedAction() == null + || action.getUnpublishedAction().getDatasource() == null) { + return false; + } + + boolean reverseFlag = StringUtils.hasText(action.getUnpublishedAction() + .getDatasource() + .getId()) + || !PluginConstants.DEFAULT_REST_DATASOURCE.equals(action.getUnpublishedAction() + .getDatasource() + .getName()); + + return !reverseFlag; + }) + .collectMap(NewAction::getGitSyncId); + }) + .map(newActionMap -> { + MigrationHelperMethods.migrateApplicationJsonToVersionTen(applicationJson, newActionMap); + return applicationJson; + }) + .switchIfEmpty(contingencyMigrationJson) + .onErrorResume(error -> { + log.error("Error occurred while migrating actions of application json. {}", error.getMessage()); + return contingencyMigrationJson; + }); + } +} diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerImplTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerImplTest.java index bf49502f059..29265607ffd 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerImplTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerImplTest.java @@ -442,7 +442,8 @@ public void autoCommitServerMigration_WhenServerHasNoChanges_NoCommitMade() thro doReturn(Mono.just(applicationJson1)) .when(jsonSchemaMigration) - .migrateApplicationJsonToLatestSchema(applicationJson); + .migrateApplicationJsonToLatestSchema( + Mockito.eq(applicationJson), Mockito.anyString(), Mockito.anyString()); doReturn(Mono.just("success")) .when(gitExecutor) @@ -574,7 +575,9 @@ public void autocommitServerMigration_WhenJsonSchemaMigrationPresent_CommitSucce AppsmithBeanUtils.copyNewFieldValuesIntoOldObject(applicationJson, applicationJson1); applicationJson1.setServerSchemaVersion(jsonSchemaVersions.getServerVersion() + 1); - doReturn(Mono.just(applicationJson1)).when(jsonSchemaMigration).migrateApplicationJsonToLatestSchema(any()); + doReturn(Mono.just(applicationJson1)) + .when(jsonSchemaMigration) + .migrateApplicationJsonToLatestSchema(any(), Mockito.anyString(), Mockito.anyString()); gitFileSystemTestHelper.setupGitRepository(autoCommitEvent, applicationJson); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java index 1c53aa243bb..e18f8e66fe9 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitServiceTest.java @@ -284,7 +284,8 @@ public void testAutoCommit_whenOnlyServerIsEligibleForMigration_commitSuccess() doReturn(Mono.just(applicationJson1)) .when(jsonSchemaMigration) - .migrateApplicationJsonToLatestSchema(any(ApplicationJson.class)); + .migrateApplicationJsonToLatestSchema( + any(ApplicationJson.class), Mockito.anyString(), Mockito.anyString()); gitFileSystemTestHelper.setupGitRepository( WORKSPACE_ID, DEFAULT_APP_ID, BRANCH_NAME, REPO_NAME, applicationJson); @@ -571,7 +572,8 @@ public void testAutoCommit_whenAutoCommitEligibleButPrerequisiteNotComplete_retu doReturn(Mono.just(applicationJson1)) .when(jsonSchemaMigration) - .migrateApplicationJsonToLatestSchema(any(ApplicationJson.class)); + .migrateApplicationJsonToLatestSchema( + any(ApplicationJson.class), Mockito.anyString(), Mockito.anyString()); gitFileSystemTestHelper.setupGitRepository( WORKSPACE_ID, DEFAULT_APP_ID, BRANCH_NAME, REPO_NAME, applicationJson); @@ -644,7 +646,8 @@ public void testAutoCommit_whenServerIsRunningMigrationCallsAutocommitAgainOnDif doReturn(Mono.just(applicationJson1)) .when(jsonSchemaMigration) - .migrateApplicationJsonToLatestSchema(any(ApplicationJson.class)); + .migrateApplicationJsonToLatestSchema( + any(ApplicationJson.class), Mockito.anyString(), Mockito.anyString()); gitFileSystemTestHelper.setupGitRepository( WORKSPACE_ID, DEFAULT_APP_ID, BRANCH_NAME, REPO_NAME, applicationJson); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/migrations/JsonSchemaMigrationTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/migrations/JsonSchemaMigrationTest.java index d31c18fcdcb..9fbbfbe6120 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/migrations/JsonSchemaMigrationTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/migrations/JsonSchemaMigrationTest.java @@ -97,7 +97,7 @@ public void migrateApplicationJsonToLatestSchema_whenFeatureFlagIsOn_returnsIncr gitFileSystemTestHelper.getApplicationJson(this.getClass().getResource("application.json")); Mono applicationJsonMono = - jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson); + jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson, null, null); StepVerifier.create(applicationJsonMono) .assertNext(appJson -> { assertThat(appJson.getServerSchemaVersion()).isEqualTo(jsonSchemaVersions.getServerVersion()); @@ -121,7 +121,7 @@ public void migrateApplicationJsonToLatestSchema_whenFeatureFlagIsOff_returnsFal gitFileSystemTestHelper.getApplicationJson(this.getClass().getResource("application.json")); Mono applicationJsonMono = - jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson); + jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson, null, null); StepVerifier.create(applicationJsonMono) .assertNext(appJson -> { assertThat(appJson.getClientSchemaVersion()).isEqualTo(jsonSchemaVersions.getClientVersion());