Skip to content

Commit

Permalink
Add planning optimizing status for tables
Browse files Browse the repository at this point in the history
  • Loading branch information
huyuanfeng committed Nov 22, 2023
1 parent 5436d1e commit 1c8974e
Show file tree
Hide file tree
Showing 28 changed files with 50 additions and 24 deletions.
0 ...es/static/css/chunk-3900a95a.73a728c3.css → ...es/static/css/chunk-01d27b8f.73a728c3.css
View file
File renamed without changes.
0 ...es/static/css/chunk-9fc4147c.d250cc66.css → ...es/static/css/chunk-1c88c5da.d250cc66.css
View file
File renamed without changes.
0 ...es/static/css/chunk-f8e3eb7e.30a759b2.css → ...es/static/css/chunk-4b3af723.30a759b2.css
View file
File renamed without changes.
2 changes: 1 addition & 1 deletion ...es/static/css/chunk-cc7f866c.3e429678.css → ...es/static/css/chunk-a830a942.f7fa6324.css
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions ams/dashboard/src/main/resources/static/css/chunk-vendors.ac2f20f8.css
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion ams/dashboard/src/main/resources/static/css/chunk-vendors.d101f0ee.css
View file

This file was deleted.

2 changes: 1 addition & 1 deletion ams/dashboard/src/main/resources/static/index.html
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
<!DOCTYPE html><html lang="zh-CN"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/favicon.ico"><title>Amoro</title><link href="/css/chunk-225a1827.9a01f178.css" rel="prefetch"><link href="/css/chunk-28b9a9cc.1a34fd26.css" rel="prefetch"><link href="/css/chunk-3900a95a.73a728c3.css" rel="prefetch"><link href="/css/chunk-6a44215d.d87fcd67.css" rel="prefetch"><link href="/css/chunk-76776ef2.413b8385.css" rel="prefetch"><link href="/css/chunk-7c2b5c40.1594e78b.css" rel="prefetch"><link href="/css/chunk-9fc4147c.d250cc66.css" rel="prefetch"><link href="/css/chunk-cc7f866c.3e429678.css" rel="prefetch"><link href="/css/chunk-e27313ec.e6dc7a03.css" rel="prefetch"><link href="/css/chunk-eb399e28.3a56beaf.css" rel="prefetch"><link href="/css/chunk-f0807d3c.0d6cdd01.css" rel="prefetch"><link href="/css/chunk-f8e3eb7e.30a759b2.css" rel="prefetch"><link href="/js/chunk-225a1827.6e4a0bcf.js" rel="prefetch"><link href="/js/chunk-28b9a9cc.8ad4d357.js" rel="prefetch"><link href="/js/chunk-2d0a3577.6e005997.js" rel="prefetch"><link href="/js/chunk-2d0a43df.78514714.js" rel="prefetch"><link href="/js/chunk-2d0aa90c.416ebc57.js" rel="prefetch"><link href="/js/chunk-2d0aab07.05af786a.js" rel="prefetch"><link href="/js/chunk-2d0abc00.5f3e5163.js" rel="prefetch"><link href="/js/chunk-2d0ae937.d9152499.js" rel="prefetch"><link href="/js/chunk-2d0aeb45.7bb5a4a4.js" rel="prefetch"><link href="/js/chunk-2d0afa49.dddb2605.js" rel="prefetch"><link href="/js/chunk-2d0b2762.32c299e8.js" rel="prefetch"><link href="/js/chunk-2d0b6187.ce1d2a89.js" rel="prefetch"><link href="/js/chunk-2d0ba136.69c3a813.js" rel="prefetch"><link href="/js/chunk-2d0bb267.9c2f6321.js" rel="prefetch"><link href="/js/chunk-2d0bff92.b0873be3.js" rel="prefetch"><link href="/js/chunk-2d0c0494.39b4894f.js" rel="prefetch"><link href="/js/chunk-2d0c0a09.e2276bd3.js" rel="prefetch"><link href="/js/chunk-2d0c1ed0.c977c8b3.js" rel="prefetch"><link href="/js/chunk-2d0c4313.570158d3.js" rel="prefetch"><link href="/js/chunk-2d0c46d1.fe7a4f31.js" rel="prefetch"><link href="/js/chunk-2d0c512b.1f8f3dfd.js" rel="prefetch"><link href="/js/chunk-2d0cf16e.50e1fba1.js" rel="prefetch"><link href="/js/chunk-2d0d056d.042799e4.js" rel="prefetch"><link href="/js/chunk-2d0d0645.111e3b36.js" rel="prefetch"><link href="/js/chunk-2d0d2f22.c0862770.js" rel="prefetch"><link href="/js/chunk-2d0d7e63.5906e0f3.js" rel="prefetch"><link href="/js/chunk-2d0e1b57.ee01262c.js" rel="prefetch"><link href="/js/chunk-2d0e1fbe.301c0d37.js" rel="prefetch"><link href="/js/chunk-2d0e22d6.22b9ea0a.js" rel="prefetch"><link href="/js/chunk-2d0e542a.303b9da0.js" rel="prefetch"><link href="/js/chunk-2d0e57ec.ea372c29.js" rel="prefetch"><link href="/js/chunk-2d0e6553.9633aaad.js" rel="prefetch"><link href="/js/chunk-2d0e6c86.7d85c1d7.js" rel="prefetch"><link href="/js/chunk-2d0ea098.37ad26fd.js" rel="prefetch"><link href="/js/chunk-2d208ac5.c1639251.js" rel="prefetch"><link href="/js/chunk-2d209408.80802248.js" rel="prefetch"><link href="/js/chunk-2d20f745.9063d79b.js" rel="prefetch"><link href="/js/chunk-2d20ff23.caebe04c.js" rel="prefetch"><link href="/js/chunk-2d2138c7.dc7f0fa2.js" rel="prefetch"><link href="/js/chunk-2d216f3b.5cb150e6.js" rel="prefetch"><link href="/js/chunk-2d217e5b.a87a35b6.js" rel="prefetch"><link href="/js/chunk-2d21b84a.9729acd0.js" rel="prefetch"><link href="/js/chunk-2d21dcd2.0bb15a8c.js" rel="prefetch"><link href="/js/chunk-2d21f327.6a54a4ed.js" rel="prefetch"><link href="/js/chunk-2d2214b3.28dc7f93.js" rel="prefetch"><link href="/js/chunk-2d221814.a3436a84.js" rel="prefetch"><link href="/js/chunk-2d221a34.1c09c805.js" rel="prefetch"><link href="/js/chunk-2d22502a.f074e7b3.js" rel="prefetch"><link href="/js/chunk-2d228ca6.6cf6352c.js" rel="prefetch"><link href="/js/chunk-2d2295e9.89d90cf5.js" rel="prefetch"><link href="/js/chunk-2d22c171.ed835096.js" rel="prefetch"><link href="/js/chunk-2d22c2b8.55b6c439.js" rel="prefetch"><link href="/js/chunk-2d22ca58.ddb7682f.js" rel="prefetch"><link href="/js/chunk-2d2311f7.b9d7efe4.js" rel="prefetch"><link href="/js/chunk-2d2371be.88831637.js" rel="prefetch"><link href="/js/chunk-2d237ee7.49f75c8b.js" rel="prefetch"><link href="/js/chunk-2d238465.9afee280.js" rel="prefetch"><link href="/js/chunk-3900a95a.39f8ad9f.js" rel="prefetch"><link href="/js/chunk-6a44215d.7bf9a929.js" rel="prefetch"><link href="/js/chunk-746e09c3.3a1e61db.js" rel="prefetch"><link href="/js/chunk-7532b3ea.eef96a1d.js" rel="prefetch"><link href="/js/chunk-76776ef2.fe6037ed.js" rel="prefetch"><link href="/js/chunk-7c2b5c40.c33ef31e.js" rel="prefetch"><link href="/js/chunk-9fc4147c.668c8945.js" rel="prefetch"><link href="/js/chunk-cc7f866c.6b6888fb.js" rel="prefetch"><link href="/js/chunk-e27313ec.2d3c24aa.js" rel="prefetch"><link href="/js/chunk-eb399e28.aba60b35.js" rel="prefetch"><link href="/js/chunk-f0807d3c.6646b85f.js" rel="prefetch"><link href="/js/chunk-f8e3eb7e.bd955d07.js" rel="prefetch"><link href="/css/app.e88a00e5.css" rel="preload" as="style"><link href="/css/chunk-vendors.d101f0ee.css" rel="preload" as="style"><link href="/js/app.599de2c8.js" rel="preload" as="script"><link href="/js/chunk-vendors.6eab6434.js" rel="preload" as="script"><link href="/css/chunk-vendors.d101f0ee.css" rel="stylesheet"><link href="/css/app.e88a00e5.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but Arctic doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/js/chunk-vendors.6eab6434.js"></script><script src="/js/app.599de2c8.js"></script></body></html>
<!DOCTYPE html><html lang="zh-CN"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/favicon.ico"><title>Amoro</title><link href="/css/chunk-01d27b8f.73a728c3.css" rel="prefetch"><link href="/css/chunk-1c88c5da.d250cc66.css" rel="prefetch"><link href="/css/chunk-225a1827.9a01f178.css" rel="prefetch"><link href="/css/chunk-28b9a9cc.1a34fd26.css" rel="prefetch"><link href="/css/chunk-4b3af723.30a759b2.css" rel="prefetch"><link href="/css/chunk-6a44215d.d87fcd67.css" rel="prefetch"><link href="/css/chunk-76776ef2.413b8385.css" rel="prefetch"><link href="/css/chunk-7c2b5c40.1594e78b.css" rel="prefetch"><link href="/css/chunk-a830a942.f7fa6324.css" rel="prefetch"><link href="/css/chunk-e27313ec.e6dc7a03.css" rel="prefetch"><link href="/css/chunk-eb399e28.3a56beaf.css" rel="prefetch"><link href="/css/chunk-f0807d3c.0d6cdd01.css" rel="prefetch"><link href="/js/chunk-01d27b8f.bd5867b0.js" rel="prefetch"><link href="/js/chunk-1c88c5da.d9b18f40.js" rel="prefetch"><link href="/js/chunk-225a1827.cee53f42.js" rel="prefetch"><link href="/js/chunk-28b9a9cc.bdd7a368.js" rel="prefetch"><link href="/js/chunk-2d0a3577.6e005997.js" rel="prefetch"><link href="/js/chunk-2d0a43df.78514714.js" rel="prefetch"><link href="/js/chunk-2d0aa90c.416ebc57.js" rel="prefetch"><link href="/js/chunk-2d0aab07.05af786a.js" rel="prefetch"><link href="/js/chunk-2d0abc00.5f3e5163.js" rel="prefetch"><link href="/js/chunk-2d0ae937.d9152499.js" rel="prefetch"><link href="/js/chunk-2d0aeb45.7bb5a4a4.js" rel="prefetch"><link href="/js/chunk-2d0afa49.dddb2605.js" rel="prefetch"><link href="/js/chunk-2d0b2762.32c299e8.js" rel="prefetch"><link href="/js/chunk-2d0b6187.ce1d2a89.js" rel="prefetch"><link href="/js/chunk-2d0ba136.69c3a813.js" rel="prefetch"><link href="/js/chunk-2d0bb267.9c2f6321.js" rel="prefetch"><link href="/js/chunk-2d0bff92.b0873be3.js" rel="prefetch"><link href="/js/chunk-2d0c0494.39b4894f.js" rel="prefetch"><link href="/js/chunk-2d0c0a09.e2276bd3.js" rel="prefetch"><link href="/js/chunk-2d0c1ed0.c977c8b3.js" rel="prefetch"><link href="/js/chunk-2d0c4313.570158d3.js" rel="prefetch"><link href="/js/chunk-2d0c46d1.fe7a4f31.js" rel="prefetch"><link href="/js/chunk-2d0c512b.1f8f3dfd.js" rel="prefetch"><link href="/js/chunk-2d0cf16e.50e1fba1.js" rel="prefetch"><link href="/js/chunk-2d0d056d.042799e4.js" rel="prefetch"><link href="/js/chunk-2d0d0645.111e3b36.js" rel="prefetch"><link href="/js/chunk-2d0d2f22.c0862770.js" rel="prefetch"><link href="/js/chunk-2d0d7e63.5906e0f3.js" rel="prefetch"><link href="/js/chunk-2d0e1b57.ee01262c.js" rel="prefetch"><link href="/js/chunk-2d0e1fbe.301c0d37.js" rel="prefetch"><link href="/js/chunk-2d0e22d6.22b9ea0a.js" rel="prefetch"><link href="/js/chunk-2d0e542a.303b9da0.js" rel="prefetch"><link href="/js/chunk-2d0e57ec.ea372c29.js" rel="prefetch"><link href="/js/chunk-2d0e6553.9633aaad.js" rel="prefetch"><link href="/js/chunk-2d0e6c86.7d85c1d7.js" rel="prefetch"><link href="/js/chunk-2d0ea098.37ad26fd.js" rel="prefetch"><link href="/js/chunk-2d208ac5.c1639251.js" rel="prefetch"><link href="/js/chunk-2d209408.80802248.js" rel="prefetch"><link href="/js/chunk-2d20f745.9063d79b.js" rel="prefetch"><link href="/js/chunk-2d20ff23.caebe04c.js" rel="prefetch"><link href="/js/chunk-2d2138c7.dc7f0fa2.js" rel="prefetch"><link href="/js/chunk-2d216f3b.5cb150e6.js" rel="prefetch"><link href="/js/chunk-2d217e5b.a87a35b6.js" rel="prefetch"><link href="/js/chunk-2d21b84a.9729acd0.js" rel="prefetch"><link href="/js/chunk-2d21dcd2.0bb15a8c.js" rel="prefetch"><link href="/js/chunk-2d21f327.6a54a4ed.js" rel="prefetch"><link href="/js/chunk-2d2214b3.28dc7f93.js" rel="prefetch"><link href="/js/chunk-2d221814.a3436a84.js" rel="prefetch"><link href="/js/chunk-2d221a34.1c09c805.js" rel="prefetch"><link href="/js/chunk-2d22502a.f074e7b3.js" rel="prefetch"><link href="/js/chunk-2d228ca6.6cf6352c.js" rel="prefetch"><link href="/js/chunk-2d2295e9.89d90cf5.js" rel="prefetch"><link href="/js/chunk-2d22c171.ed835096.js" rel="prefetch"><link href="/js/chunk-2d22c2b8.55b6c439.js" rel="prefetch"><link href="/js/chunk-2d22ca58.ddb7682f.js" rel="prefetch"><link href="/js/chunk-2d2311f7.b9d7efe4.js" rel="prefetch"><link href="/js/chunk-2d2371be.88831637.js" rel="prefetch"><link href="/js/chunk-2d237ee7.49f75c8b.js" rel="prefetch"><link href="/js/chunk-2d238465.9afee280.js" rel="prefetch"><link href="/js/chunk-4b3af723.c1542276.js" rel="prefetch"><link href="/js/chunk-6a44215d.182511f3.js" rel="prefetch"><link href="/js/chunk-746e09c3.3a1e61db.js" rel="prefetch"><link href="/js/chunk-7532b3ea.eef96a1d.js" rel="prefetch"><link href="/js/chunk-76776ef2.945b676f.js" rel="prefetch"><link href="/js/chunk-7c2b5c40.7e68252c.js" rel="prefetch"><link href="/js/chunk-a830a942.e0232fb1.js" rel="prefetch"><link href="/js/chunk-e27313ec.28b609cb.js" rel="prefetch"><link href="/js/chunk-eb399e28.5dfb4488.js" rel="prefetch"><link href="/js/chunk-f0807d3c.7419103d.js" rel="prefetch"><link href="/css/app.e88a00e5.css" rel="preload" as="style"><link href="/css/chunk-vendors.ac2f20f8.css" rel="preload" as="style"><link href="/js/app.ae1f911f.js" rel="preload" as="script"><link href="/js/chunk-vendors.6eab6434.js" rel="preload" as="script"><link href="/css/chunk-vendors.ac2f20f8.css" rel="stylesheet"><link href="/css/app.e88a00e5.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but Arctic doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div><script src="/js/chunk-vendors.6eab6434.js"></script><script src="/js/app.ae1f911f.js"></script></body></html>
2 changes: 1 addition & 1 deletion .../main/resources/static/js/app.599de2c8.js → .../main/resources/static/js/app.ae1f911f.js
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions ...rces/static/js/chunk-3900a95a.39f8ad9f.js → ...rces/static/js/chunk-01d27b8f.bd5867b0.js
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions ams/dashboard/src/main/resources/static/js/chunk-1c88c5da.d9b18f40.js
View file

Large diffs are not rendered by default.

0 ...rces/static/js/chunk-225a1827.6e4a0bcf.js → ...rces/static/js/chunk-225a1827.cee53f42.js
View file
File renamed without changes.
0 ...rces/static/js/chunk-28b9a9cc.8ad4d357.js → ...rces/static/js/chunk-28b9a9cc.bdd7a368.js
View file
File renamed without changes.
1 change: 1 addition & 0 deletions ams/dashboard/src/main/resources/static/js/chunk-4b3af723.c1542276.js
View file

Large diffs are not rendered by default.

0 ...rces/static/js/chunk-6a44215d.7bf9a929.js → ...rces/static/js/chunk-6a44215d.182511f3.js
View file
File renamed without changes.
0 ...rces/static/js/chunk-76776ef2.fe6037ed.js → ...rces/static/js/chunk-76776ef2.945b676f.js
View file
File renamed without changes.
0 ...rces/static/js/chunk-7c2b5c40.c33ef31e.js → ...rces/static/js/chunk-7c2b5c40.7e68252c.js
View file
File renamed without changes.
1 change: 0 additions & 1 deletion ams/dashboard/src/main/resources/static/js/chunk-9fc4147c.668c8945.js
View file

This file was deleted.

1 change: 1 addition & 0 deletions ams/dashboard/src/main/resources/static/js/chunk-a830a942.e0232fb1.js
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion ams/dashboard/src/main/resources/static/js/chunk-cc7f866c.6b6888fb.js
View file

This file was deleted.

0 ...rces/static/js/chunk-e27313ec.2d3c24aa.js → ...rces/static/js/chunk-e27313ec.28b609cb.js
View file
File renamed without changes.
0 ...rces/static/js/chunk-eb399e28.aba60b35.js → ...rces/static/js/chunk-eb399e28.5dfb4488.js
View file
File renamed without changes.
0 ...rces/static/js/chunk-f0807d3c.6646b85f.js → ...rces/static/js/chunk-f0807d3c.7419103d.js
View file
File renamed without changes.
1 change: 0 additions & 1 deletion ams/dashboard/src/main/resources/static/js/chunk-f8e3eb7e.bd955d07.js
View file

This file was deleted.

1 change: 1 addition & 0 deletions ams/dashboard/src/views/optimize/components/List.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ const emit = defineEmits<{
}>()
const STATUS_CONFIG = shallowReactive({
pending: { title: 'pending', color: '#ffcc00' },
planning: { title: 'planning', color: '#076de3' },
idle: { title: 'idle', color: '#c9cdd4' },
minor: { title: 'minor', color: '#0ad787' },
major: { title: 'major', color: '#0ad787' },
Expand Down
1 change: 1 addition & 0 deletions ams/dashboard/src/views/resource/components/List.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ const emit = defineEmits<{(e: 'editGroup', record: IIOptimizeGroupItem): void; (
const STATUS_CONFIG = shallowReactive({
pending: { title: 'pending', color: '#ffcc00' },
planning: { title: 'planning', color: '#076de3' },
idle: { title: 'idle', color: '#c9cdd4' },
minor: { title: 'minor', color: '#0ad787' },
major: { title: 'major', color: '#0ad787' },
Expand Down
23 changes: 11 additions & 12 deletions ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingQueue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,7 @@ private void initTableRuntime(TableRuntimeMeta tableRuntimeMeta) {
// TODO: load task quotas
tableRuntime.resetTaskQuotas(
System.currentTimeMillis() - ArcticServiceConstants.QUOTA_LOOK_BACK_TIME);
if (tableRuntime.getOptimizingStatus() == OptimizingStatus.IDLE
|| tableRuntime.getOptimizingStatus() == OptimizingStatus.PENDING) {
if (!tableRuntime.getOptimizingStatus().isProcessing()) {
schedulingPolicy.addTable(tableRuntime);
} else if (tableRuntime.getOptimizingStatus() != OptimizingStatus.COMMITTING) {
TableOptimizingProcess process = new TableOptimizingProcess(tableRuntimeMeta);
Expand Down Expand Up @@ -358,17 +357,16 @@ private void planTasks() {
List<TableIdentifier> plannedTables = Lists.newArrayList();
for (TableRuntime tableRuntime : scheduledTables) {
LOG.debug("Planning table {}", tableRuntime.getTableIdentifier());
AmoroTable<?> table = tableManager.loadTable(tableRuntime.getTableIdentifier());
OptimizingPlanner planner =
new OptimizingPlanner(
tableRuntime.refresh(table), (ArcticTable) table.originalTable(), getAvailableCore());
if (tableRuntime.isBlocked(BlockableOperation.OPTIMIZE)) {
LOG.info("{} optimize is blocked, continue", tableRuntime.getTableIdentifier());
continue;
}
tableRuntime.beginPlanning();
try {
AmoroTable<?> table = tableManager.loadTable(tableRuntime.getTableIdentifier());
OptimizingPlanner planner =
new OptimizingPlanner(
tableRuntime.refresh(table),
(ArcticTable) table.originalTable(),
getAvailableCore());
if (tableRuntime.isBlocked(BlockableOperation.OPTIMIZE)) {
LOG.info("{} optimize is blocked, continue", tableRuntime.getTableIdentifier());
continue;
}
plannedTables.add(table.id());
if (planner.isNecessary()) {
TableOptimizingProcess optimizingProcess = new TableOptimizingProcess(planner);
Expand All @@ -382,6 +380,7 @@ private void planTasks() {
tableRuntime.cleanPendingInput();
}
} catch (Throwable e) {
tableRuntime.planFailed();
LOG.error(tableRuntime.getTableIdentifier() + " plan failed, continue", e);
}
}
Expand Down
1 change: 1 addition & 0 deletions ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingStatus.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ public enum OptimizingStatus {
MAJOR_OPTIMIZING("major", true),
MINOR_OPTIMIZING("minor", true),
COMMITTING("committing", true),
PLANNING("planning", false),
PENDING("pending", false),
IDLE("idle", false);
private final String displayValue;
Expand Down
Loading

0 comments on commit 1c8974e

Please sign in to comment.