From ef696df082a441491bd09eb8df2f6f75c5afdecd Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 7 Nov 2023 14:30:58 +0800 Subject: [PATCH 1/8] do not check sys tenant user. --- pom.xml | 2 +- .../odc/service/schedule/DlmEnvironment.java | 23 ------------- .../flowtask/DataArchivePreprocessor.java | 9 ----- .../flowtask/DataDeletePreprocessor.java | 7 ---- .../schedule/DesktopDlmEnvironment.java | 34 ------------------- .../service/schedule/WebDlmEnvironment.java | 34 ------------------- 6 files changed, 1 insertion(+), 108 deletions(-) delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/DlmEnvironment.java delete mode 100644 server/starters/desktop-starter/src/main/java/com/oceanbase/odc/service/schedule/DesktopDlmEnvironment.java delete mode 100644 server/starters/web-starter/src/main/java/com/oceanbase/odc/service/schedule/WebDlmEnvironment.java diff --git a/pom.xml b/pom.xml index 50f6080c9d..9df2b5ee16 100644 --- a/pom.xml +++ b/pom.xml @@ -116,7 +116,7 @@ 2.1.6 - 1.0.4 + 1.0.6-SNAPSHOT 2.11.0 diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/DlmEnvironment.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/DlmEnvironment.java deleted file mode 100644 index 91374736fb..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/DlmEnvironment.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.schedule; - -public interface DlmEnvironment { - - boolean isSysTenantUserRequired(); - -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java index 2e167e4198..0769ccf9d3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataArchivePreprocessor.java @@ -34,7 +34,6 @@ import com.oceanbase.odc.service.flow.processor.ScheduleTaskPreprocessor; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; -import com.oceanbase.odc.service.schedule.DlmEnvironment; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.model.JobType; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; @@ -49,10 +48,6 @@ @Slf4j @ScheduleTaskPreprocessor(type = JobType.DATA_ARCHIVE) public class DataArchivePreprocessor extends AbstractDlmJobPreprocessor { - - @Autowired - private DlmEnvironment dlmEnvironment; - @Autowired private AuthenticationFacade authenticationFacade; @@ -75,10 +70,6 @@ public void process(CreateFlowInstanceReq req) { // permission to access it. Database sourceDb = databaseService.detail(dataArchiveParameters.getSourceDatabaseId()); Database targetDb = databaseService.detail(dataArchiveParameters.getTargetDataBaseId()); - if (dlmEnvironment.isSysTenantUserRequired()) { - checkDatasource(sourceDb.getDataSource()); - checkDatasource(targetDb.getDataSource()); - } dataArchiveParameters.setSourceDatabaseName(sourceDb.getName()); dataArchiveParameters.setTargetDatabaseName(targetDb.getName()); dataArchiveParameters.setSourceDataSourceName(sourceDb.getDataSource().getName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java index eae06dc4e1..1965362da6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java @@ -29,7 +29,6 @@ import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq; import com.oceanbase.odc.service.flow.processor.ScheduleTaskPreprocessor; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; -import com.oceanbase.odc.service.schedule.DlmEnvironment; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.model.JobType; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; @@ -45,9 +44,6 @@ @ScheduleTaskPreprocessor(type = JobType.DATA_DELETE) public class DataDeletePreprocessor extends AbstractDlmJobPreprocessor { - @Autowired - private DlmEnvironment dlmEnvironment; - @Autowired private AuthenticationFacade authenticationFacade; @@ -69,9 +65,6 @@ public void process(CreateFlowInstanceReq req) { // Throw exception when the specified database does not exist or the current user does not have // permission to access it. Database sourceDb = databaseService.detail(dataDeleteParameters.getDatabaseId()); - if (dlmEnvironment.isSysTenantUserRequired()) { - checkDatasource(sourceDb.getDataSource()); - } ConnectionConfig dataSource = sourceDb.getDataSource(); dataSource.setDefaultSchema(sourceDb.getName()); diff --git a/server/starters/desktop-starter/src/main/java/com/oceanbase/odc/service/schedule/DesktopDlmEnvironment.java b/server/starters/desktop-starter/src/main/java/com/oceanbase/odc/service/schedule/DesktopDlmEnvironment.java deleted file mode 100644 index 4dac5d9670..0000000000 --- a/server/starters/desktop-starter/src/main/java/com/oceanbase/odc/service/schedule/DesktopDlmEnvironment.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.schedule; - -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Service; - -import com.oceanbase.odc.core.authority.util.SkipAuthorize; - -@Service -@Profile("clientMode") -@SkipAuthorize("odc internal usage") -public class DesktopDlmEnvironment implements DlmEnvironment { - - @Override - public boolean isSysTenantUserRequired() { - return true; - } - -} diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/schedule/WebDlmEnvironment.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/schedule/WebDlmEnvironment.java deleted file mode 100644 index 88f2373ee3..0000000000 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/schedule/WebDlmEnvironment.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.schedule; - -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Service; - -import com.oceanbase.odc.core.authority.util.SkipAuthorize; - -@Service -@Profile("alipay") -@SkipAuthorize("odc internal usage") -public class WebDlmEnvironment implements DlmEnvironment { - - @Override - public boolean isSysTenantUserRequired() { - return true; - } - -} From 89206124fdd95dad2b4e7c7ef098bfffd39f6cd1 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 8 Nov 2023 15:26:38 +0800 Subject: [PATCH 2/8] fix data delete failed after data archived. --- .../odc/service/schedule/job/DataArchiveDeleteJob.java | 4 ++++ .../odc/service/schedule/job/DataArchiveRollbackJob.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java index b3a949675a..e104e88290 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java @@ -25,6 +25,7 @@ import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.service.dlm.model.DlmTask; +import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; import com.oceanbase.odc.service.schedule.model.DataArchiveClearParameters; import com.oceanbase.tools.migrator.common.enums.JobType; @@ -68,6 +69,9 @@ public void execute(JobExecutionContext context) { List taskUnits = JsonUtils.fromJson(dataArchiveTask.getResultJson(), new TypeReference>() {}); taskUnits.forEach(taskUnit -> { + taskUnit.setId(DlmJobIdUtil.generateHistoryJobId(taskEntity.getJobName(), taskEntity.getJobGroup(), + taskEntity.getId(), + taskUnits.size())); taskUnit.setJobType(JobType.DELETE); taskUnit.setStatus(TaskStatus.PREPARING); }); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java index a2da96e85c..ec319c4ea2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java @@ -25,6 +25,7 @@ import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.service.dlm.model.DlmTask; +import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; import com.oceanbase.odc.service.schedule.model.DataArchiveRollbackParameters; import com.oceanbase.tools.migrator.common.enums.JobType; @@ -65,6 +66,9 @@ public void execute(JobExecutionContext context) { new TypeReference>() {}); taskUnits.forEach(taskUnit -> { Long temp = taskUnit.getSourceDatabaseId(); + taskUnit.setId(DlmJobIdUtil.generateHistoryJobId(taskEntity.getJobName(), taskEntity.getJobGroup(), + taskEntity.getId(), + taskUnits.size())); taskUnit.setSourceDatabaseId(taskUnit.getTargetDatabaseId()); taskUnit.setTargetDatabaseId(temp); taskUnit.setJobType(JobType.ROLLBACK); From 2c878541740ce3f965d517dacdb6d82172198cb2 Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 14 Nov 2023 15:59:22 +0800 Subject: [PATCH 3/8] fix delete data failed. --- .../odc/service/schedule/ScheduleService.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 1efa729d63..84bfdb2968 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -88,6 +88,7 @@ import com.oceanbase.odc.service.schedule.model.ScheduleTaskMapper; import com.oceanbase.odc.service.schedule.model.ScheduleTaskResp; import com.oceanbase.odc.service.schedule.model.TriggerConfig; +import com.oceanbase.odc.service.schedule.model.TriggerStrategy; import com.oceanbase.odc.service.task.model.ExecutorInfo; import lombok.extern.slf4j.Slf4j; @@ -304,16 +305,16 @@ public ScheduleTaskResp dataArchiveDelete(Long scheduleId, Long taskId) { // TODO throw try { - if (!quartzJobService.checkExists(jobKey)) { - CreateQuartzJobReq req = new CreateQuartzJobReq(); - req.setScheduleId(scheduleId); - req.setType(JobType.DATA_ARCHIVE_DELETE); - DataArchiveClearParameters parameters = new DataArchiveClearParameters(); - parameters.setDataArchiveTaskId(taskId); - req.getJobDataMap().putAll(BeanMap.create(parameters)); - quartzJobService.createJob(req); - } - quartzJobService.triggerJob(jobKey); + CreateQuartzJobReq req = new CreateQuartzJobReq(); + req.setScheduleId(scheduleId); + req.setType(JobType.DATA_ARCHIVE_DELETE); + DataArchiveClearParameters parameters = new DataArchiveClearParameters(); + parameters.setDataArchiveTaskId(taskId); + req.getJobDataMap().putAll(BeanMap.create(parameters)); + TriggerConfig triggerConfig = new TriggerConfig(); + triggerConfig.setTriggerStrategy(TriggerStrategy.START_NOW); + req.setTriggerConfig(triggerConfig); + quartzJobService.createJob(req); } catch (SchedulerException e) { throw new RuntimeException(e); } From ff15e86222cc879d4a9a57bde654b8c09288ec1c Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 14 Nov 2023 16:33:58 +0800 Subject: [PATCH 4/8] fix delete data failed. --- .../com/oceanbase/odc/service/schedule/ScheduleService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 84bfdb2968..02df4daee3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -305,14 +305,17 @@ public ScheduleTaskResp dataArchiveDelete(Long scheduleId, Long taskId) { // TODO throw try { + if (!quartzJobService.checkExists(jobKey)) { + quartzJobService.deleteJob(jobKey); + } CreateQuartzJobReq req = new CreateQuartzJobReq(); req.setScheduleId(scheduleId); req.setType(JobType.DATA_ARCHIVE_DELETE); DataArchiveClearParameters parameters = new DataArchiveClearParameters(); parameters.setDataArchiveTaskId(taskId); - req.getJobDataMap().putAll(BeanMap.create(parameters)); TriggerConfig triggerConfig = new TriggerConfig(); triggerConfig.setTriggerStrategy(TriggerStrategy.START_NOW); + req.getJobDataMap().putAll(BeanMap.create(parameters)); req.setTriggerConfig(triggerConfig); quartzJobService.createJob(req); } catch (SchedulerException e) { From 60f5df4e904be38c111efb16c8828065bca2c46d Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 14 Nov 2023 16:54:02 +0800 Subject: [PATCH 5/8] fix delete data failed. --- .../com/oceanbase/odc/service/schedule/ScheduleService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 02df4daee3..d8548581a2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -305,7 +305,7 @@ public ScheduleTaskResp dataArchiveDelete(Long scheduleId, Long taskId) { // TODO throw try { - if (!quartzJobService.checkExists(jobKey)) { + if (quartzJobService.checkExists(jobKey)) { quartzJobService.deleteJob(jobKey); } CreateQuartzJobReq req = new CreateQuartzJobReq(); From 5babeae101b61ad9fdbb264633c2da66ead136dc Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 21 Nov 2023 14:22:45 +0800 Subject: [PATCH 6/8] fix update limit config failed. --- .../odc/service/schedule/flowtask/DataDeletePreprocessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java index 6d92c8b3c5..7e4e791c9a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DataDeletePreprocessor.java @@ -91,7 +91,7 @@ public void process(CreateFlowInstanceReq req) { if (dataDeleteParameters.getRateLimit().getDataSizeLimit() != null) { limiterConfig.setDataSizeLimit(dataDeleteParameters.getRateLimit().getDataSizeLimit()); } - if (dataDeleteParameters.getRateLimit().getRowLimit() != null) { + if (dataDeleteParameters.getRateLimit().getBatchSize() != null) { limiterConfig.setBatchSize(dataDeleteParameters.getRateLimit().getBatchSize()); } limiterService.createAndBindToOrder(scheduleEntity.getId(), limiterConfig); From 7f60fe4d1ba6e40354b78fbb81752d58b64645c8 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 22 Nov 2023 11:32:58 +0800 Subject: [PATCH 7/8] fix rollback failed. --- .../odc/service/schedule/ScheduleService.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index d8548581a2..21e6ef69c2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -303,9 +303,9 @@ public ScheduleTaskResp dataArchiveDelete(Long scheduleId, Long taskId) { throw new IllegalStateException("Delete is not allowed because the data archive job has not succeeded."); } - // TODO throw try { if (quartzJobService.checkExists(jobKey)) { + log.info("Data archive delete job exists and start delete job,jobKey={}", jobKey); quartzJobService.deleteJob(jobKey); } CreateQuartzJobReq req = new CreateQuartzJobReq(); @@ -341,16 +341,20 @@ public ScheduleTaskResp rollbackTask(Long scheduleId, Long taskId) { } try { - if (!quartzJobService.checkExists(jobKey)) { - CreateQuartzJobReq req = new CreateQuartzJobReq(); - req.setScheduleId(scheduleId); - req.setType(JobType.DATA_ARCHIVE_ROLLBACK); - DataArchiveRollbackParameters parameters = new DataArchiveRollbackParameters(); - parameters.setDataArchiveTaskId(taskId); - req.getJobDataMap().putAll(BeanMap.create(parameters)); - quartzJobService.createJob(req); + if (quartzJobService.checkExists(jobKey)) { + log.info("Data archive rollback job exists and start delete job,jobKey={}", jobKey); + quartzJobService.deleteJob(jobKey); } - quartzJobService.triggerJob(jobKey); + CreateQuartzJobReq req = new CreateQuartzJobReq(); + req.setScheduleId(scheduleId); + req.setType(JobType.DATA_ARCHIVE_ROLLBACK); + DataArchiveRollbackParameters parameters = new DataArchiveRollbackParameters(); + parameters.setDataArchiveTaskId(taskId); + req.getJobDataMap().putAll(BeanMap.create(parameters)); + TriggerConfig triggerConfig = new TriggerConfig(); + triggerConfig.setTriggerStrategy(TriggerStrategy.START_NOW); + req.setTriggerConfig(triggerConfig); + quartzJobService.createJob(req); } catch (SchedulerException e) { throw new RuntimeException(e); } From de6258baa5f189a81c62d6aaa4df0484b712732e Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 22 Nov 2023 11:33:26 +0800 Subject: [PATCH 8/8] upgrade dlm sdk to 106 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 96cb15d747..d378b4d05d 100644 --- a/pom.xml +++ b/pom.xml @@ -116,7 +116,7 @@ 2.1.6 - 1.0.6-SNAPSHOT + 1.0.6 2.11.0