diff --git a/.gitignore b/.gitignore index fdc9d45b2c..1cea7d58a8 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ lib/odc-*.jar .mvn/ maven/ venv/ +**/local-unit-test.properties diff --git a/.secignore b/.secignore index ca42c3126d..946958eb65 100644 --- a/.secignore +++ b/.secignore @@ -88,12 +88,6 @@ -------------------------------------------------------- # Should use GLOB wildcard to configure and analysis the ignored folder # The root patch should start with '/' -/test-script/** -/script/functions.sh -/builds/** -/server/integration-test/** -/server/odc-common/src/test/resources/resource/** -/server/odc-common/target/** /client/** /log/** /log/async/** @@ -101,21 +95,14 @@ /logs/** /data/** /maven/** -/.antcode/** /.idea/** /.git/** -/README.md /.gitmodules /.gitemplate /.secignore -/build_daily_aci.yml -/build_release_aci.yml /secretScanResult /server/integration-test/src/test/resources/ssl/ca.pem -/libs/ob-sql-parser/README.md /libs/ob-sql-parser/src/main/resources/** -/README-zh-CN.md -/docs/** /server/integration-test/src/test/java/com/oceanbase/odc/service/connection/DefaultCertificateStorageTest.java -------------------------------------------------------- # Set the ignored fold to escape the Chinese scan by GLOB wildcard diff --git a/builds/unit-test.properties b/builds/unit-test.properties deleted file mode 100644 index 585632f16f..0000000000 --- a/builds/unit-test.properties +++ /dev/null @@ -1,11 +0,0 @@ -# Unit test OB cluster environment configuration. -# Sensitive value auto encrypt by com.oceanbase.odc.test.tool.EncryptableConfigurations. -# While change value, just input plain text and the value will be replaced to encrypted one while first time loaded -# (e.g. execute any unit test class using com.oceanbase.odc.test.database.TestDBConfigurations) -odc.ob.default.oracle.commandline=ENC@ -odc.ob.default.oracle.sysUsername=ENC@ -odc.ob.default.oracle.sysPassword=ENC@ -odc.ob.default.mysql.commandline=ENC@ -odc.ob.default.mysql.sysUsername=ENC@ -odc.ob.default.mysql.sysPassword=ENC@ -odc.mysql.default.commandline=ENC@ diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/env/BasePropertiesEnv.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/env/BasePropertiesEnv.java index dd3ede973f..1f5960828b 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/env/BasePropertiesEnv.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/env/BasePropertiesEnv.java @@ -46,7 +46,7 @@ @Slf4j public abstract class BasePropertiesEnv { - private static final String TEST_CONFIG_FILE = "../../builds/unit-test.properties"; + private static final String TEST_CONFIG_FILE = "../../local-unit-test.properties"; private static final String ENCRYPTED_PREFIX = "ENC@"; private static final Properties PROPERTIES = new Properties(); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/ITConfigurations.java b/server/integration-test/src/test/java/com/oceanbase/odc/ITConfigurations.java deleted file mode 100644 index a29b85238d..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/ITConfigurations.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc; - -import java.util.Map; - -import com.oceanbase.odc.common.config.EncryptableConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration.CloudProvider; -import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; -import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties.OmsProperties; - -/** - * integration test configurations - */ -public class ITConfigurations { - private static final String CONFIG_FILE = "src/test/resources/integration-test.properties"; - private static final Map properties = EncryptableConfigurations.loadProperties(CONFIG_FILE); - - public static String getOb311OracleCommandLine() { - return get("odc.ob.311.oracle.commandline"); - } - - public static ObjectStorageConfiguration getOssConfiguration() { - ObjectStorageConfiguration configuration = new ObjectStorageConfiguration(); - configuration.setCloudProvider(CloudProvider.ALIBABA_CLOUD); - configuration.setRegion(get("odc.cloud.object-storage.oss.region")); - configuration.setEndpoint(get("odc.cloud.object-storage.oss.endpoint")); - configuration.setAccessKeyId(get("odc.cloud.object-storage.oss.access-key-id")); - configuration.setAccessKeySecret(get("odc.cloud.object-storage.oss.access-key-secret")); - configuration.setBucketName(get("odc.cloud.object-storage.oss.bucket-name")); - configuration.setRoleArn(get("odc.cloud.object-storage.oss.role-arn")); - configuration.setRoleSessionName(get("odc.cloud.object-storage.oss.role-session-name")); - return configuration; - } - - public static ObjectStorageConfiguration getS3Configuration() { - ObjectStorageConfiguration configuration = new ObjectStorageConfiguration(); - configuration.setCloudProvider(CloudProvider.AWS); - configuration.setRegion(get("odc.cloud.object-storage.s3.region")); - configuration.setEndpoint(get("odc.cloud.object-storage.s3.endpoint")); - configuration.setAccessKeyId(get("odc.cloud.object-storage.s3.access-key-id")); - configuration.setAccessKeySecret(get("odc.cloud.object-storage.s3.access-key-secret")); - configuration.setBucketName(get("odc.cloud.object-storage.s3.bucket-name")); - configuration.setRoleArn(get("odc.cloud.object-storage.s3.role-arn")); - configuration.setRoleSessionName(get("odc.cloud.object-storage.s3.role-session-name")); - return configuration; - } - - public static OnlineSchemaChangeProperties getOscPrivateCloudProperties() { - OnlineSchemaChangeProperties configuration = new OnlineSchemaChangeProperties(); - OmsProperties omsConfiguration = new OmsProperties(); - omsConfiguration.setUrl(get("odc.osc.oms.url")); - omsConfiguration.setAuthorization(get("odc.osc.oms.authorization")); - omsConfiguration.setRegion(get("odc.osc.oms.region")); - configuration.setOms(omsConfiguration); - return configuration; - } - - private static String get(String key) { - return properties.get(key); - } - -} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/AWSSecurityTokenServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/AWSSecurityTokenServiceTest.java deleted file mode 100644 index 874b6d929e..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/AWSSecurityTokenServiceTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.cloud; - -import java.io.File; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.amazonaws.auth.AWSSessionCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.oceanbase.odc.ITConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceService; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceService.GenerateTempCredentialReq; -import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.objectstorage.cloud.model.UploadObjectTemporaryCredential; - -public class AWSSecurityTokenServiceTest { - private static final String FILE_NAME = "src/test/resources/data/test0001.txt"; - private static final String FILE_NAME_CN_ZH = "src/test/resources/data/中文名称.txt"; - private static ObjectStorageConfiguration configuration; - private static CloudResourceService service; - - @BeforeClass - public static void beforeClass() { - configuration = ITConfigurations.getS3Configuration(); - CloudClient cloudClient = new CloudResourceConfigurations().cloudClient(() -> configuration); - CloudObjectStorageService cloudObjectStorageService = - new CloudObjectStorageService(cloudClient, () -> configuration); - service = new CloudResourceService(cloudObjectStorageService, cloudClient); - } - - @Test - public void generateTempCredential() { - generateTempCredential("test0001.txt", FILE_NAME); - } - - @Test - public void generateTempCredential_CNZH() { - generateTempCredential("中文名称.txt", FILE_NAME_CN_ZH); - } - - public void generateTempCredential(String fileName, String filePath) { - GenerateTempCredentialReq req = new GenerateTempCredentialReq(); - req.setFileName(fileName); - UploadObjectTemporaryCredential credential = service.generateTempCredential(req); - - AWSSessionCredentials sessionCredentials = new BasicSessionCredentials( - credential.getAccessKeyId(), credential.getAccessKeySecret(), - credential.getSecurityToken()); - - AmazonS3 s3 = AmazonS3ClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(sessionCredentials)) - .withRegion(credential.getRegion()) - .disableChunkedEncoding() - .build(); - - PutObjectResult putObjectResult = - s3.putObject(configuration.getBucketName(), credential.getFilePath(), new File(filePath)); - - Assert.assertNotNull(putObjectResult.getETag()); - } - - -} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/AlibabaCloudSecurityTokenServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/AlibabaCloudSecurityTokenServiceTest.java deleted file mode 100644 index 55d6e27e4e..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/AlibabaCloudSecurityTokenServiceTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.cloud; - -import java.io.File; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; -import com.aliyun.oss.model.PutObjectResult; -import com.oceanbase.odc.ITConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceService; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceService.GenerateTempCredentialReq; -import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.objectstorage.cloud.model.UploadObjectTemporaryCredential; - -@Ignore -public class AlibabaCloudSecurityTokenServiceTest { - private static final String FILE_NAME = "src/test/resources/data/test0001.txt"; - private static final String FILE_NAME_CN_ZH = "src/test/resources/data/中文名称.txt"; - private static ObjectStorageConfiguration configuration; - private static CloudResourceService service; - - @BeforeClass - public static void beforeClass() { - configuration = ITConfigurations.getOssConfiguration(); - CloudClient cloudClient = new CloudResourceConfigurations().cloudClient(() -> configuration); - CloudObjectStorageService cloudObjectStorageService = - new CloudObjectStorageService(cloudClient, () -> configuration); - service = new CloudResourceService(cloudObjectStorageService, cloudClient); - } - - @Test - public void generateTempCredential() { - generateTempCredential("test0001.txt", FILE_NAME); - } - - @Test - public void generateTempCredential_CNZH() { - generateTempCredential("中文名称.txt", FILE_NAME_CN_ZH); - } - - private void generateTempCredential(String fileName, String filepath) { - GenerateTempCredentialReq req = new GenerateTempCredentialReq(); - req.setFileName(fileName); - UploadObjectTemporaryCredential credential = service.generateTempCredential(req); - - OSS oss = new OSSClientBuilder().build(credential.getEndpoint(), credential.getAccessKeyId(), - credential.getAccessKeySecret(), credential.getSecurityToken()); - PutObjectResult putObjectResult = - oss.putObject(configuration.getBucketName(), credential.getFilePath(), new File(filepath)); - - Assert.assertEquals(putObjectResult.getClientCRC(), putObjectResult.getServerCRC()); - } - -} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/OSSObjcetStorageServiceIT.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/OSSObjcetStorageServiceIT.java deleted file mode 100644 index 09b655d60b..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/OSSObjcetStorageServiceIT.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.cloud; - -import com.oceanbase.odc.ITConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; - -public class OSSObjcetStorageServiceIT extends AbstractCloudObjectStorageServiceTest { - @Override - CloudObjectStorageService createCloudObjectStorageService() { - ObjectStorageConfiguration configuration = ITConfigurations.getS3Configuration(); - CloudClient cloudClient = new CloudResourceConfigurations().cloudClient(() -> configuration); - return new CloudObjectStorageService(cloudClient, () -> configuration); - } -} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/OmsOpenApiServiceIT.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/OmsOpenApiServiceIT.java deleted file mode 100644 index 5b0d2c1d96..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/OmsOpenApiServiceIT.java +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.cloud; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.awaitility.Awaitility; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.SingleConnectionDataSource; - -import com.oceanbase.odc.ITConfigurations; -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.config.RestTemplateConfig; -import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; -import com.oceanbase.odc.service.onlineschemachange.oms.client.DefaultPrivateCloudOmsClient; -import com.oceanbase.odc.service.onlineschemachange.oms.client.OmsClient; -import com.oceanbase.odc.service.onlineschemachange.oms.enums.CheckerObjectStatus; -import com.oceanbase.odc.service.onlineschemachange.oms.enums.CheckerResultType; -import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsOceanBaseType; -import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsStepName; -import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsStepStatus; -import com.oceanbase.odc.service.onlineschemachange.oms.enums.ProjectStatusEnum; -import com.oceanbase.odc.service.onlineschemachange.oms.openapi.DataSourceOpenApiService; -import com.oceanbase.odc.service.onlineschemachange.oms.openapi.DataSourceOpenApiServiceImpl; -import com.oceanbase.odc.service.onlineschemachange.oms.openapi.ProjectOpenApiService; -import com.oceanbase.odc.service.onlineschemachange.oms.openapi.ProjectOpenApiServiceImpl; -import com.oceanbase.odc.service.onlineschemachange.oms.request.CommonTransferConfig; -import com.oceanbase.odc.service.onlineschemachange.oms.request.CreateOceanBaseDataSourceRequest; -import com.oceanbase.odc.service.onlineschemachange.oms.request.CreateProjectRequest; -import com.oceanbase.odc.service.onlineschemachange.oms.request.DatabaseTransferObject; -import com.oceanbase.odc.service.onlineschemachange.oms.request.FullTransferConfig; -import com.oceanbase.odc.service.onlineschemachange.oms.request.IncrTransferConfig; -import com.oceanbase.odc.service.onlineschemachange.oms.request.ListProjectFullVerifyResultRequest; -import com.oceanbase.odc.service.onlineschemachange.oms.request.ProjectControlRequest; -import com.oceanbase.odc.service.onlineschemachange.oms.request.SpecificTransferMapping; -import com.oceanbase.odc.service.onlineschemachange.oms.request.TableTransferObject; -import com.oceanbase.odc.service.onlineschemachange.oms.response.FullVerifyTableStatisticVO; -import com.oceanbase.odc.service.onlineschemachange.oms.response.ProjectFullVerifyResultResponse; -import com.oceanbase.odc.service.onlineschemachange.oms.response.ProjectProgressResponse; -import com.oceanbase.odc.service.onlineschemachange.oms.response.ProjectStepVO; -import com.oceanbase.odc.test.database.TestDBConfiguration; -import com.oceanbase.odc.test.database.TestDBConfigurations; -import com.oceanbase.odc.test.database.TestDBUtil; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author yaobin - * @date 2023-06-02 - * @since 4.2.0 - */ -@Ignore("manual verify only, may cost plenty of time and cause oms working thread exhausted") -@Slf4j -public class OmsOpenApiServiceIT { - - private static DataSourceOpenApiService dataSourceOpenApiService; - private static ProjectOpenApiService projectOpenApiService; - - private static TestDBConfiguration config; - private static JdbcTemplate jdbcTemplate; - - private String datasourceId; - private String projectId; - - @BeforeClass - public static void beforeClass() { - config = TestDBConfigurations.getInstance().getTestOBMysqlConfiguration(); - jdbcTemplate = new JdbcTemplate(config.getDataSource()); - OnlineSchemaChangeProperties configuration = ITConfigurations.getOscPrivateCloudProperties(); - OmsClient omsClient = new DefaultPrivateCloudOmsClient(configuration, - new RestTemplateConfig().omsRestTemplate()); - dataSourceOpenApiService = new DataSourceOpenApiServiceImpl(omsClient); - projectOpenApiService = new ProjectOpenApiServiceImpl(omsClient); - } - - @AfterClass - public static void afterClass() { - dropTable(); - } - - @Test - public void test_oms_openapi() { - test_create_datasource(); - try { - test_project_start(); - test_project_migrate(); - test_listProjectFullVerifyResult(); - test_project_control(); - } finally { - test_project_release_and_delete(); - } - } - - private void test_create_datasource() { - CreateOceanBaseDataSourceRequest request = getCreateOceanBaseDataSourceRequest(); - - datasourceId = dataSourceOpenApiService.createOceanBaseDataSource(request); - Assert.assertNotNull("datasourceId is empty", datasourceId); - } - - private void test_project_start() { - createTable(); - - CreateProjectRequest createProjectRequest = getCreateProjectRequest(); - projectId = projectOpenApiService.createProject(createProjectRequest); - Assert.assertNotNull(projectId); - - ProjectControlRequest request = getProjectControlRequest(projectId); - ProjectProgressResponse projectProgressResponse = projectOpenApiService.describeProjectProgress(request); - projectOpenApiService.startProject(request); - Assert.assertEquals(ProjectStatusEnum.INIT, projectProgressResponse.getStatus()); - projectProgressResponse = projectOpenApiService.describeProjectProgress(request); - Assert.assertEquals(ProjectStatusEnum.RUNNING, projectProgressResponse.getStatus()); - } - - private void test_project_migrate() { - - updateTable(); - - checkStepFinished(OmsStepName.TRANSFER_PRECHECK, Duration.ofSeconds(30)); - checkStepFinished(OmsStepName.TRANSFER_INCR_LOG_PULL, Duration.ofSeconds(90)); - checkStepFinished(OmsStepName.FULL_TRANSFER, Duration.ofSeconds(60)); - checkStepFinished(OmsStepName.INCR_TRANSFER, Duration.ofSeconds(30)); - checkStepFinished(OmsStepName.FULL_VERIFIER, Duration.ofSeconds(60)); - - } - - private void checkStepFinished(OmsStepName omsStepName, Duration duration) { - Awaitility.with().pollInterval(Duration.ofSeconds(5)) - .and().pollDelay(Duration.ofSeconds(5)) - .and().atMost(duration) - .await("wait for oms execute over, check step name is " + omsStepName.name()) - .until(() -> { - List projectStepsVO = - projectOpenApiService.describeProjectSteps(getProjectControlRequest(projectId)); - // test get step status cycle - Map projectStepMap = - projectStepsVO.stream().collect(Collectors.toMap(ProjectStepVO::getName, a -> a)); - log.info("oms return result: {}", JsonUtils.toJson(projectStepMap)); - Assert.assertNotNull(projectStepsVO); - return checkStepFinished(projectStepMap, omsStepName); - - }); - } - - private void test_listProjectFullVerifyResult() { - ListProjectFullVerifyResultRequest request = new ListProjectFullVerifyResultRequest(); - request.setProjectId(projectId); - request.setSourceSchemas(new String[] {config.getDefaultDBName()}); - request.setDestSchemas(new String[] {config.getDefaultDBName()}); - request.setStatus(new String[] {"FINISHED", "SUSPEND", "RUNNING"}); - request.setPageSize(10); - request.setPageNumber(1); - - ProjectFullVerifyResultResponse verifyResultResponse = - projectOpenApiService.listProjectFullVerifyResult(request); - - Assert.assertNotNull(verifyResultResponse.getDifferentNumber()); - Assert.assertEquals(0, verifyResultResponse.getDifferentNumber().longValue()); - List fullVerifyTableStatistics = - verifyResultResponse.getFullVerifyTableStatistics(); - Assert.assertTrue(fullVerifyTableStatistics.size() > 0); - Assert.assertEquals(CheckerObjectStatus.FINISHED, fullVerifyTableStatistics.get(0).getStatus()); - Assert.assertEquals(CheckerResultType.CONSISTENT, fullVerifyTableStatistics.get(0).getResultType()); - - } - - private boolean checkStepFinished(Map projectStepMap, OmsStepName name) { - Assert.assertNotNull(projectStepMap.get(name)); - OmsStepStatus status = projectStepMap.get(name).getStatus(); - Integer progress = projectStepMap.get(name).getProgress(); - - Function competedFunc = (p -> p != null && p == 100); - - switch (name) { - case INCR_TRANSFER: - return status == OmsStepStatus.MONITORING && competedFunc.apply(progress); - case FULL_VERIFIER: - return status == OmsStepStatus.RUNNING && competedFunc.apply(progress); - default: - return status == OmsStepStatus.FINISHED && competedFunc.apply(progress); - } - - } - - private void test_project_control() { - ProjectControlRequest request = getProjectControlRequest(projectId); - - projectOpenApiService.stopProject(request); - ProjectProgressResponse projectProgressResponse = projectOpenApiService.describeProjectProgress(request); - Assert.assertEquals(ProjectStatusEnum.SUSPEND, projectProgressResponse.getStatus()); - - projectOpenApiService.resumeProject(request); - projectProgressResponse = projectOpenApiService.describeProjectProgress(request); - Assert.assertEquals(ProjectStatusEnum.RUNNING, projectProgressResponse.getStatus()); - - projectOpenApiService.stopProject(request); - projectProgressResponse = projectOpenApiService.describeProjectProgress(request); - Assert.assertEquals(ProjectStatusEnum.SUSPEND, projectProgressResponse.getStatus()); - - } - - private void test_project_release_and_delete() { - if (projectId == null) { - return; - } - - ProjectControlRequest request = getProjectControlRequest(projectId); - - ProjectProgressResponse projectProgressResponse = projectOpenApiService.describeProjectProgress(request); - if (projectProgressResponse.getStatus() == ProjectStatusEnum.RUNNING) { - projectOpenApiService.stopProject(request); - } - - projectOpenApiService.releaseProject(request); - projectProgressResponse = projectOpenApiService.describeProjectProgress(request); - Assert.assertTrue(ProjectStatusEnum.RELEASING == projectProgressResponse.getStatus() || - ProjectStatusEnum.RELEASED == projectProgressResponse.getStatus()); - - Awaitility.with().pollInterval(Duration.ofSeconds(5)) - .and().pollDelay(Duration.ofSeconds(5)) - .and().atMost(Duration.ofSeconds(70)) - .await("wait for oms released project") - .until(() -> { - ProjectProgressResponse response = projectOpenApiService.describeProjectProgress(request); - return response.getStatus() == ProjectStatusEnum.RELEASED; - - }); - - projectOpenApiService.deleteProject(request); - } - - private void createTable() { - - jdbcTemplate.execute("create table t1(id int(20) primary key, name1 varchar(20))"); - jdbcTemplate.execute("insert into t1 values(1,'abc'),(2,'efg')"); - jdbcTemplate.execute("create table t1_gho(id int(20) primary key, name1 varchar(20), name2 varchar(20))"); - } - - private void updateTable() { - jdbcTemplate.execute("insert into t1 values(3,'abc'),(4,'efg')"); - jdbcTemplate.execute("update t1 set id = 5 where id = 3"); - jdbcTemplate.execute("delete from t1 where id = 2"); - } - - private static void dropTable() { - jdbcTemplate.execute("drop table if exists t1"); - jdbcTemplate.execute("drop table if exists t1_gho"); - } - - private CreateProjectRequest getCreateProjectRequest() { - CreateProjectRequest request = new CreateProjectRequest(); - request.setSourceEndpointId(datasourceId); - request.setSinkEndpointId(datasourceId); - - List databaseTransferObjects = new ArrayList<>(); - DatabaseTransferObject databaseTransferObject = new DatabaseTransferObject(); - databaseTransferObject.setName(config.getDefaultDBName()); - databaseTransferObject.setMappedName(config.getDefaultDBName()); - databaseTransferObjects.add(databaseTransferObject); - - List tables = new ArrayList<>(); - TableTransferObject tableTransferObject = new TableTransferObject(); - tableTransferObject.setName("t1"); - tableTransferObject.setMappedName("t1_gho"); - tables.add(tableTransferObject); - databaseTransferObject.setTables(tables); - - SpecificTransferMapping transferMapping = new SpecificTransferMapping(); - transferMapping.setDatabases(databaseTransferObjects); - request.setTransferMapping(transferMapping); - - CommonTransferConfig commonTransferConfig = new CommonTransferConfig(); - request.setCommonTransferConfig(commonTransferConfig); - - FullTransferConfig fullTransferConfig = new FullTransferConfig(); - request.setFullTransferConfig(fullTransferConfig); - - IncrTransferConfig incrTransferConfig = new IncrTransferConfig(); - request.setIncrTransferConfig(incrTransferConfig); - - request.setEnableFullVerify(Boolean.TRUE); - request.setName("IT-" + StringUtils.uuidNoHyphen()); - return request; - } - - private CreateOceanBaseDataSourceRequest getCreateOceanBaseDataSourceRequest() { - CreateOceanBaseDataSourceRequest request = new CreateOceanBaseDataSourceRequest(); - request.setName("IT-" + StringUtils.uuidNoHyphen()); - request.setType(OmsOceanBaseType.OB_MYSQL.name()); - request.setTenant(config.getTenant()); - request.setCluster(config.getCluster()); - request.setVpcId(null); - request.setIp(config.getHost()); - request.setPort(config.getPort()); - request.setUserName(config.getUsername()); - request.setPassword(Base64.getEncoder().encodeToString(config.getSysPassword().getBytes())); - - request.setRegion("cn-anhui"); - request.setDescription(null); - request.setOcpName(null); - - String configUrl = getConfigUrl(); - request.setConfigUrl(configUrl); - request.setDrcUserName(config.getSysUsername()); - request.setDrcPassword(Base64.getEncoder().encodeToString(config.getSysPassword().getBytes())); - return request; - } - - private String getConfigUrl() { - String queryClusterUrlSql = "show parameters like 'obconfig_url'"; - String configUrl; - try (Connection connection = DriverManager.getConnection( - TestDBUtil.buildUrl(config.getHost(), config.getPort(), config.getDefaultDBName(), "OB_MYSQL"), - TestDBUtil.buildUser(config.getSysUsername(), config.getTenant(), config.getCluster()), - config.getSysPassword())) { - configUrl = new JdbcTemplate(new SingleConnectionDataSource(connection, false)) - .query(queryClusterUrlSql, rs -> { - if (!rs.next()) { - throw new IllegalArgumentException("Get ob config_url is empty"); - } - return rs.getString("value"); - }); - - } catch (SQLException e) { - throw new RuntimeException(e); - } - return configUrl; - } - - private ProjectControlRequest getProjectControlRequest(String projectId) { - ProjectControlRequest request = new ProjectControlRequest(); - request.setId(projectId); - return request; - } - -} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/S3ObjcetStorageServiceIT.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/S3ObjcetStorageServiceIT.java deleted file mode 100644 index d105332936..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/cloud/S3ObjcetStorageServiceIT.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.cloud; - -import com.oceanbase.odc.ITConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceConfigurations; -import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; - -public class S3ObjcetStorageServiceIT extends AbstractCloudObjectStorageServiceTest { - @Override - CloudObjectStorageService createCloudObjectStorageService() { - ObjectStorageConfiguration configuration = ITConfigurations.getS3Configuration(); - CloudClient cloudClient = new CloudResourceConfigurations().cloudClient(() -> configuration); - return new CloudObjectStorageService(cloudClient, () -> configuration); - } -} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewerTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewerTest.java index ea9359d079..8157536674 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewerTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/connection/ConnectionBatchImportPreviewerTest.java @@ -104,7 +104,7 @@ public void success_Simplified_Chinese_PreviewBatchImportConnection() throws IOE ConnectionPreviewBatchImportResp req; LocaleContextHolder.setLocale(Locale.SIMPLIFIED_CHINESE); req = connectionBatchImportPreviewer.preview(multipartFile_Simplified_Chinese); - Assert.assertEquals(8, req.getBatchImportConnectionList().size()); + Assert.assertEquals(4, req.getBatchImportConnectionList().size()); } @Test diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/lab/ResourceServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/lab/ResourceServiceTest.java deleted file mode 100644 index 779b28ebd1..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/lab/ResourceServiceTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.lab; - -import static org.mockito.Mockito.when; - -import java.util.Arrays; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.test.jdbc.JdbcTestUtils; - -import com.oceanbase.odc.ServiceTestEnv; -import com.oceanbase.odc.common.config.EncryptableConfigurations; -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; -import com.oceanbase.odc.service.collaboration.environment.model.Environment; -import com.oceanbase.odc.service.collaboration.project.ProjectService; -import com.oceanbase.odc.service.iam.UserService; -import com.oceanbase.odc.service.iam.model.CreateUserReq; -import com.oceanbase.odc.service.iam.model.User; -import com.oceanbase.odc.service.lab.model.LabProperties; - -public class ResourceServiceTest extends ServiceTestEnv { - - private static final String TEST_CONFIG_FILE = "src/test/resources/lab-test.properties"; - @Autowired - private ResourceService resourceService; - @Autowired - private LabDataSourceFactory labDataSourceFactory; - @Autowired - private UserService userService; - @Autowired - private JdbcTemplate jdbcTemplate; - @Autowired - private TestLabProperties testLabProperties; - @MockBean - private LabProperties labProperties; - @MockBean - private ProjectService projectService; - @MockBean - private EnvironmentService environmentService; - - private User user; - - @Before - public void setUp() { - CreateUserReq createUserReq = new CreateUserReq(); - createUserReq.setName("test"); - createUserReq.setAccountName("test"); - createUserReq.setPassword("test"); - createUserReq.setEnabled(true); - this.user = userService.create(createUserReq); - - when(labProperties.isLabEnabled()).thenReturn(true); - when(labProperties.getInitMysqlResourceInitScriptTemplate()) - .thenReturn(testLabProperties.getInitMysqlResourceInitScriptTemplate()); - when(labProperties.getInitMysqlResourceRevokeScriptTemplate()) - .thenReturn(testLabProperties.getInitMysqlResourceRevokeScriptTemplate()); - when(labProperties.getObConnectionKey()).thenReturn( - JsonUtils.toJson(Arrays.asList(EncryptableConfigurations.loadProperties(TEST_CONFIG_FILE)))); - when(environmentService.detailSkipPermissionCheck(Mockito.anyLong())).thenReturn(getEnvironment()); - when(environmentService.exists(Mockito.anyLong())).thenReturn(true); - - - labDataSourceFactory.init(); - } - - @After - public void tearDown() throws Exception { - if (this.user != null) { - userService.delete(this.user.getId()); - } - } - - @Test - @Ignore - public void createThenRevoke() { - resourceService.createResource(this.user); - - int connectionCount = JdbcTestUtils.countRowsInTableWhere(jdbcTemplate, - "connect_connection", "`name` like 'oceanbase_mysql_test_%'"); - Assert.assertEquals(1, connectionCount); - - Long connectionId = jdbcTemplate.queryForObject( - "select `id` from connect_connection where `name` like 'oceanbase_mysql_test_%'", - Long.class); - - boolean ret = resourceService.revokeResource(connectionId); - Assert.assertTrue(ret); - } - - private Environment getEnvironment() { - Environment environment = new Environment(); - environment.setId(1L); - environment.setName("fake_env"); - return environment; - } - -} diff --git a/server/integration-test/src/test/resources/batchImport/connection/connection_template_simplified_chinese.xlsx b/server/integration-test/src/test/resources/batchImport/connection/connection_template_simplified_chinese.xlsx index 443993e7a3..ec1fcc6649 100644 Binary files a/server/integration-test/src/test/resources/batchImport/connection/connection_template_simplified_chinese.xlsx and b/server/integration-test/src/test/resources/batchImport/connection/connection_template_simplified_chinese.xlsx differ diff --git a/server/integration-test/src/test/resources/batchImport/connection/connection_template_traditional_chinese.xlsx b/server/integration-test/src/test/resources/batchImport/connection/connection_template_traditional_chinese.xlsx index f21f82b2e7..a117d607c6 100644 Binary files a/server/integration-test/src/test/resources/batchImport/connection/connection_template_traditional_chinese.xlsx and b/server/integration-test/src/test/resources/batchImport/connection/connection_template_traditional_chinese.xlsx differ diff --git a/server/integration-test/src/test/resources/batchImport/connection/connection_template_us.xlsx b/server/integration-test/src/test/resources/batchImport/connection/connection_template_us.xlsx index b3c24999db..6c04aa8992 100644 Binary files a/server/integration-test/src/test/resources/batchImport/connection/connection_template_us.xlsx and b/server/integration-test/src/test/resources/batchImport/connection/connection_template_us.xlsx differ diff --git a/server/integration-test/src/test/resources/batchImport/user/user_template_simplified_chinese.xlsx b/server/integration-test/src/test/resources/batchImport/user/user_template_simplified_chinese.xlsx index ce5ad986a4..0b08386879 100644 Binary files a/server/integration-test/src/test/resources/batchImport/user/user_template_simplified_chinese.xlsx and b/server/integration-test/src/test/resources/batchImport/user/user_template_simplified_chinese.xlsx differ diff --git a/server/integration-test/src/test/resources/batchImport/user/user_template_traditional_chinese.xlsx b/server/integration-test/src/test/resources/batchImport/user/user_template_traditional_chinese.xlsx index 6d903e2816..7b56bb9177 100644 Binary files a/server/integration-test/src/test/resources/batchImport/user/user_template_traditional_chinese.xlsx and b/server/integration-test/src/test/resources/batchImport/user/user_template_traditional_chinese.xlsx differ diff --git a/server/integration-test/src/test/resources/batchImport/user/user_template_us.xlsx b/server/integration-test/src/test/resources/batchImport/user/user_template_us.xlsx index e0983a776f..dcd4297696 100644 Binary files a/server/integration-test/src/test/resources/batchImport/user/user_template_us.xlsx and b/server/integration-test/src/test/resources/batchImport/user/user_template_us.xlsx differ diff --git a/server/integration-test/src/test/resources/integration-test.properties b/server/integration-test/src/test/resources/integration-test.properties deleted file mode 100644 index 7127182d96..0000000000 --- a/server/integration-test/src/test/resources/integration-test.properties +++ /dev/null @@ -1,46 +0,0 @@ -# sensitive value auto encrypt by EncryptableConfigurations -# while change value, just input plain text, -# the value will be replaced to encrypted one while first time loaded(e.g. execute any unit test) -#use cli command line for create test connection -#e.g. odc.integration-test.ob311=obclient -h localhost -P2881 -test@test#test -pyourpassword -odc.ob.311.oracle.commandline=ENC@FlnKSFNsluMq9thS+6bXd9gC38r/1qLqw2oSD6S9Lx66zdx8qQnEzZJr74ynlYK/XQXdkPHDtgnLimJw2m+UPQ== -# -#alibaba cloud oss -# -odc.cloud.object-storage.oss.provider=ALIBABA_CLOUD -odc.cloud.object-storage.oss.region=cn-hangzhou -odc.cloud.object-storage.oss.endpoint= -odc.cloud.object-storage.oss.access-key-id=ENC@vvwtA1Kzcj4uihA6OvyRBWzoTxwQm6Ck -odc.cloud.object-storage.oss.access-key-secret=ENC@XcVGruqQh7MxjEMD65IjILPelsZsfYrtn01OzBeCLaI= -odc.cloud.object-storage.oss.bucket-name=cn-hangzhou-odcmc-dev -odc.cloud.object-storage.oss.role-arn=acs:ram::1936434522080725:role/odcossputonlyrole -odc.cloud.object-storage.oss.role-session-name=odctest -# -#amazon web service s3 -# -odc.cloud.object-storage.s3.provider=AWS -odc.cloud.object-storage.s3.region=us-west-2 -odc.cloud.object-storage.s3.endpoint= -odc.cloud.object-storage.s3.access-key-id=ENC@rFeEeIpOV+tW9PK9UrVXwejHf3+DekVR -odc.cloud.object-storage.s3.access-key-secret=ENC@MvO6hIjPrxeO1WmYeEnhIbeIXmQiD9blFQL17HcgwPLpXliKbe/n/A== -odc.cloud.object-storage.s3.bucket-name=us-west-2-odcmc2 -odc.cloud.object-storage.s3.role-arn=arn:aws:iam::042274075875:role/ODCS3PutOnly -odc.cloud.object-storage.s3.role-session-name=odctest - -# -#oceanbase alibaba cloud -# -odc.osc.cloud.alibaba-cloud.oms-service-provider=ALIBABA_CLOUD -odc.osc.cloud.alibaba-cloud.access-key-id=ENC@uUBKTO/1He+rUyD0RVRZPoMxV6XIIIQ1 -odc.osc.cloud.alibaba-cloud.access-key-secret=ENC@MlAUEbL5DWlk/FkKznO6v01yYxDZUTkD+JC9dRdoBBs= -odc.osc.cloud.alibaba-cloud.endpoint=ENC@prZMAcfyCaFgcppzFebGRgxKcnQbFPLmYbytVKmeR6pFaNSqsB/vlGpgFP6JSqQx -odc.osc.cloud.alibaba-cloud.region=ap-southeast-1 -odc.osc.cloud.alibaba-cloud.db-password=ENC@28BObEaCB2wW9hK6w2HI6Q== - -# -#private cloud -# -odc.osc.oms.oms-service-provider=PRIVATE_CLOUD -odc.osc.oms.url=ENC@v0/nMqcGpgdTpu8kv2xVrf+brr9LvXRt -odc.osc.oms.authorization=ENC@WkayDDVCdk4NrAHoKtic4Xub2ZdeXg6W -odc.osc.oms.region=ap-southeast-1 diff --git a/server/integration-test/src/test/resources/lab-test.properties b/server/integration-test/src/test/resources/lab-test.properties deleted file mode 100644 index 9ffa63f814..0000000000 --- a/server/integration-test/src/test/resources/lab-test.properties +++ /dev/null @@ -1,7 +0,0 @@ -# sensitive value auto encrypt by EncryptableConfigurations -# while change value, just input plain text, -# the value will be replaced to encrypted one while first time loaded(e.g. execute any unit test) -host=ENC@AZPYF+WgArKh6ygRjbyrLkSyxf12rRGv -username=ENC@wC7lQOnYLGg2nwknco2pEg== -password=ENC@OiGkiuQCW/I= -jdbcUrl=ENC@ER9ISUM/T0jcn8je+DUQ+w+O8EzOwbm1i94Hql/wdGKAmh2tYSpcJin4mV714wlBVkbaD8golplTAJloRz2Y7By4cvxw5Vq4nJgW2OfQIZ0= diff --git a/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestProperties.java b/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestProperties.java index c302714f55..43624259a0 100644 --- a/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestProperties.java +++ b/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestProperties.java @@ -35,7 +35,7 @@ public class TestProperties { URL location = TestProperties.class.getProtectionDomain().getCodeSource().getLocation(); TEST_CONFIG_FILE = Paths.get(location.toURI()) .getParent().getParent().getParent().getParent() - .resolve("builds").resolve("unit-test.properties").toString(); + .resolve("local-unit-test.properties").toString(); } catch (URISyntaxException e) { throw new IllegalStateException(e); } @@ -45,4 +45,5 @@ public class TestProperties { public static String getProperty(String key) { return properties.get(key); } + }