From bf5abb8bd654de67bfb9fc0cb4e0227e44b76ba1 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 12:36:50 +0300 Subject: [PATCH 01/52] IGNITE-22435 Add JobTarget interface --- .../apache/ignite/compute/IgniteCompute.java | 65 +++++++++++++++++++ .../org/apache/ignite/compute/JobTarget.java | 31 +++++++++ 2 files changed, 96 insertions(+) create mode 100644 modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java diff --git a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java index c0fb02869d5..808817081a8 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java @@ -40,6 +40,63 @@ * @see ComputeJob#executeAsync(JobExecutionContext, Object...) */ public interface IgniteCompute { + /** + * Submits a {@link ComputeJob} of the given class for an execution on a single node from a set of candidate nodes. + * + * @param Job result type. + * @param target Execution target. + * @param descriptor Job descriptor. + * @param args Arguments of the job. + * @return Job execution object. + */ + JobExecution submit( + JobTarget target, + JobDescriptor descriptor, + Object... args + ); + + /** + * Submits a {@link ComputeJob} of the given class for an execution on a single node from a set of candidate nodes. A shortcut for + * {@code submit(...).resultAsync()}. + * + * @param Job result type. + * @param target Execution target. + * @param descriptor Job descriptor. + * @param args Arguments of the job. + * @return Job result future. + */ + default CompletableFuture executeAsync( + JobTarget target, + JobDescriptor descriptor, + Object... args + ) { + return this.submit(target, descriptor, args).resultAsync(); + } + + /** + * Executes a {@link ComputeJob} of the given class on a single node from a set of candidate nodes. + * + * @param Job result type + * @param target Execution target. + * @param descriptor Job descriptor. + * @param args Arguments of the job. + * @return Job result. + * @throws ComputeException If there is any problem executing the job. + */ + R execute( + JobTarget target, + JobDescriptor descriptor, + Object... args + ); + + + + + /** OLD METHODS BELOW */ + + + + /** * Submits a {@link ComputeJob} of the given class for an execution on a single node from a set of candidate nodes. * @@ -205,6 +262,14 @@ R executeColocated( JobDescriptor descriptor, Object... args); + + + + /** BROADCAST METHODS BELOW - NOT AFFECTED BY ExecutionTarget */ + + + + /** * Submits a {@link ComputeJob} of the given class for an execution on all nodes in the given node set. * diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java new file mode 100644 index 00000000000..ec64a5b6a5f --- /dev/null +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -0,0 +1,31 @@ +package org.apache.ignite.compute; + +import java.util.Collection; +import org.apache.ignite.network.ClusterNode; +import org.apache.ignite.table.Tuple; +import org.apache.ignite.table.mapper.Mapper; + +/** + * Job execution target. + */ +public interface JobTarget { + static JobTarget node(ClusterNode node) { + return null; // TODO + } + + static JobTarget anyNode(ClusterNode... nodes) { + return null; // TODO + } + + static JobTarget anyNode(Collection nodes) { + return null; // TODO + } + + static JobTarget colocated(Tuple key) { + return null; // TODO + } + + static JobTarget colocated(K key, Mapper keyMapper) { + return null; // TODO + } +} From 2a53da1efcc9b1172e50fdd05666b4bdd715876f Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 12:40:13 +0300 Subject: [PATCH 02/52] IGNITE-22435 Add JobTarget interface --- .../apache/ignite/compute/IgniteCompute.java | 36 ++++++++++++------- .../org/apache/ignite/compute/JobTarget.java | 4 +-- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java index 808817081a8..75a9fe67b36 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java @@ -106,11 +106,13 @@ R execute( * @param args Arguments of the job. * @return Job execution object. */ - JobExecution submit( + default JobExecution submit( Set nodes, JobDescriptor descriptor, Object... args - ); + ) { + return submit(JobTarget.anyNode(nodes), descriptor, args); + } /** * Submits a {@link ComputeJob} of the given class for an execution on a single node from a set of candidate nodes. A shortcut for @@ -140,11 +142,13 @@ default CompletableFuture executeAsync( * @return Job result. * @throws ComputeException If there is any problem executing the job. */ - R execute( + default R execute( Set nodes, JobDescriptor descriptor, Object... args - ); + ) { + return execute(JobTarget.anyNode(nodes), descriptor, args); + } /** * Submits a job of the given class for the execution on the node where the given key is located. The node is a leader of the @@ -157,12 +161,14 @@ R execute( * @param Job result type. * @return Job execution object. */ - JobExecution submitColocated( + default JobExecution submitColocated( String tableName, Tuple key, JobDescriptor descriptor, Object... args - ); + ) { + return submit(JobTarget.colocated(tableName, key), descriptor, args); + } /** * Submits a job of the given class for the execution on the node where the given key is located. The node is a leader of the @@ -176,13 +182,15 @@ JobExecution submitColocated( * @param Job result type. * @return Job execution object. */ - JobExecution submitColocated( + default JobExecution submitColocated( String tableName, K key, Mapper keyMapper, JobDescriptor descriptor, Object... args - ); + ) { + return submit(JobTarget.colocated(tableName, key, keyMapper), descriptor, args); + } /** * Submits a job of the given class for the execution on the node where the given key is located. The node is a leader of the @@ -237,11 +245,13 @@ default CompletableFuture executeColocatedAsync( * @return Job result. * @throws ComputeException If there is any problem executing the job. */ - R executeColocated( + default R executeColocated( String tableName, Tuple key, JobDescriptor descriptor, - Object... args); + Object... args) { + return execute(JobTarget.colocated(tableName, key), descriptor, args); + } /** * Executes a job of the given class on the node where the given key is located. The node is a leader of the corresponding RAFT group. @@ -255,12 +265,14 @@ R executeColocated( * @return Job result. * @throws ComputeException If there is any problem executing the job. */ - R executeColocated( + default R executeColocated( String tableName, K key, Mapper keyMapper, JobDescriptor descriptor, - Object... args); + Object... args) { + return execute(JobTarget.colocated(tableName, key, keyMapper), descriptor, args); + } diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index ec64a5b6a5f..79c54143c3a 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -21,11 +21,11 @@ static JobTarget anyNode(Collection nodes) { return null; // TODO } - static JobTarget colocated(Tuple key) { + static JobTarget colocated(String tableName, Tuple key) { return null; // TODO } - static JobTarget colocated(K key, Mapper keyMapper) { + static JobTarget colocated(String tableName, K key, Mapper keyMapper) { return null; // TODO } } From f592173952946aa68c5bd56ca27dd9fcd94a1dac Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 13:39:04 +0300 Subject: [PATCH 03/52] wip execution targets --- .../compute/ColocatedExecutionTarget.java | 47 +++++++++++++++++++ .../org/apache/ignite/compute/JobTarget.java | 24 ++++++++-- .../apache/ignite/compute/NodesJobTarget.java | 32 +++++++++++++ .../internal/compute/IgniteComputeImpl.java | 11 +++++ 4 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java create mode 100644 modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java diff --git a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java new file mode 100644 index 00000000000..03ff856a4db --- /dev/null +++ b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compute; + +import java.util.Objects; +import org.apache.ignite.table.Tuple; +import org.apache.ignite.table.mapper.Mapper; +import org.jetbrains.annotations.Nullable; + +/** + * Colocated job execution target. + */ +public class ColocatedExecutionTarget { + private final String tableName; + + private final Object key; + + private final @Nullable Mapper keyMapper; + + public ColocatedExecutionTarget(String tableName, Object key, @Nullable Mapper keyMapper) { + Objects.requireNonNull(tableName); + Objects.requireNonNull(key); + + if (keyMapper == null && !(key instanceof Tuple)) { + throw new IllegalArgumentException("Key must be an instance of Tuple when keyMapper is not provided."); + } + + this.tableName = tableName; + this.key = key; + this.keyMapper = keyMapper; + } +} diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index 79c54143c3a..f9730eabc5a 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -1,6 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.compute; import java.util.Collection; +import java.util.List; import org.apache.ignite.network.ClusterNode; import org.apache.ignite.table.Tuple; import org.apache.ignite.table.mapper.Mapper; @@ -10,15 +28,15 @@ */ public interface JobTarget { static JobTarget node(ClusterNode node) { - return null; // TODO + return new NodesJobTarget(List.of(node)); } static JobTarget anyNode(ClusterNode... nodes) { - return null; // TODO + return new NodesJobTarget(List.of(nodes)); } static JobTarget anyNode(Collection nodes) { - return null; // TODO + return new NodesJobTarget(nodes); } static JobTarget colocated(String tableName, Tuple key) { diff --git a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java new file mode 100644 index 00000000000..9819820e6c5 --- /dev/null +++ b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compute; + +import java.util.Collection; +import org.apache.ignite.network.ClusterNode; + +/** + * Nodes-based job execution target. + */ +public class NodesJobTarget implements JobTarget { + private final Collection nodes; + + public NodesJobTarget(Collection nodes) { + this.nodes = nodes; + } +} diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index 78387092a41..b17f4f332a4 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -46,6 +46,7 @@ import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionOptions; import org.apache.ignite.compute.JobStatus; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.NodeNotFoundException; import org.apache.ignite.compute.task.MapReduceJob; import org.apache.ignite.compute.task.TaskExecution; @@ -94,6 +95,16 @@ public IgniteComputeImpl(PlacementDriver placementDriver, TopologyService topolo this.clock = clock; } + @Override + public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { + return null; + } + + @Override + public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { + return null; + } + @Override public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { Objects.requireNonNull(nodes); From 8f654c0d76ea7d752dad983990e7044a19792e2f Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 13:49:50 +0300 Subject: [PATCH 04/52] wip execution targets --- .../ignite/compute/ColocatedExecutionTarget.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java index 03ff856a4db..bd3703c9a97 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java @@ -30,9 +30,9 @@ public class ColocatedExecutionTarget { private final Object key; - private final @Nullable Mapper keyMapper; + private final @Nullable Mapper keyMapper; - public ColocatedExecutionTarget(String tableName, Object key, @Nullable Mapper keyMapper) { + public ColocatedExecutionTarget(String tableName, Object key, @Nullable Mapper keyMapper) { Objects.requireNonNull(tableName); Objects.requireNonNull(key); @@ -44,4 +44,16 @@ public ColocatedExecutionTarget(String tableName, Object key, @Nullable Mapper k this.key = key; this.keyMapper = keyMapper; } + + public String tableName() { + return tableName; + } + + public Object key() { + return key; + } + + public @Nullable Mapper keyMapper() { + return keyMapper; + } } From e7b6768946077162fb703683f815e5e371d2ceac Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 13:51:33 +0300 Subject: [PATCH 05/52] wip execution targets --- .../compute/ColocatedExecutionTarget.java | 2 +- .../org/apache/ignite/compute/JobTarget.java | 36 +++++++++++++++++-- .../apache/ignite/compute/NodesJobTarget.java | 7 ++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java index bd3703c9a97..8f8d0842ed5 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java @@ -25,7 +25,7 @@ /** * Colocated job execution target. */ -public class ColocatedExecutionTarget { +public class ColocatedExecutionTarget implements JobTarget { private final String tableName; private final Object key; diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index f9730eabc5a..32ce8ae5d7d 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -27,23 +27,55 @@ * Job execution target. */ public interface JobTarget { + /** + * Creates a job target for a specific node. + * + * @param node Node. + * @return Job target. + */ static JobTarget node(ClusterNode node) { return new NodesJobTarget(List.of(node)); } + /** + * Creates a job target for any node from the provided collection. + * + * @param nodes Collection of nodes. + * @return Job target. + */ static JobTarget anyNode(ClusterNode... nodes) { return new NodesJobTarget(List.of(nodes)); } + /** + * Creates a job target for any node from the provided collection. + * + * @param nodes Collection of nodes. + * @return Job target. + */ static JobTarget anyNode(Collection nodes) { return new NodesJobTarget(nodes); } + /** + * Creates a colocated job target for a specific table and key. + * + * @param tableName Table name. + * @param key Key. + * @return Job target. + */ static JobTarget colocated(String tableName, Tuple key) { - return null; // TODO + return new ColocatedExecutionTarget(tableName, key, null); } + /** + * Creates a colocated job target for a specific table and key with mapper. + * + * @param tableName Table name. + * @param key Key. + * @return Job target. + */ static JobTarget colocated(String tableName, K key, Mapper keyMapper) { - return null; // TODO + return new ColocatedExecutionTarget(tableName, key, keyMapper); } } diff --git a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java index 9819820e6c5..73b1175f25a 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java @@ -18,6 +18,7 @@ package org.apache.ignite.compute; import java.util.Collection; +import java.util.Objects; import org.apache.ignite.network.ClusterNode; /** @@ -27,6 +28,12 @@ public class NodesJobTarget implements JobTarget { private final Collection nodes; public NodesJobTarget(Collection nodes) { + Objects.requireNonNull(nodes); + + if (nodes.isEmpty()) { + throw new IllegalArgumentException("Nodes collection must not be empty."); + } + this.nodes = nodes; } } From 3613755e2d329e9736906eb647ec8887538bef9e Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 13:58:09 +0300 Subject: [PATCH 06/52] Fix IgniteComputeImpl --- .../org/apache/ignite/compute/JobTarget.java | 9 +++---- .../apache/ignite/compute/NodesJobTarget.java | 10 +++++--- .../internal/compute/IgniteComputeImpl.java | 25 ++++++++++++++++--- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index 32ce8ae5d7d..e8439a3de3d 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -17,8 +17,7 @@ package org.apache.ignite.compute; -import java.util.Collection; -import java.util.List; +import java.util.Set; import org.apache.ignite.network.ClusterNode; import org.apache.ignite.table.Tuple; import org.apache.ignite.table.mapper.Mapper; @@ -34,7 +33,7 @@ public interface JobTarget { * @return Job target. */ static JobTarget node(ClusterNode node) { - return new NodesJobTarget(List.of(node)); + return new NodesJobTarget(Set.of(node)); } /** @@ -44,7 +43,7 @@ static JobTarget node(ClusterNode node) { * @return Job target. */ static JobTarget anyNode(ClusterNode... nodes) { - return new NodesJobTarget(List.of(nodes)); + return new NodesJobTarget(Set.of(nodes)); } /** @@ -53,7 +52,7 @@ static JobTarget anyNode(ClusterNode... nodes) { * @param nodes Collection of nodes. * @return Job target. */ - static JobTarget anyNode(Collection nodes) { + static JobTarget anyNode(Set nodes) { return new NodesJobTarget(nodes); } diff --git a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java index 73b1175f25a..5b289b75254 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java @@ -17,17 +17,17 @@ package org.apache.ignite.compute; -import java.util.Collection; import java.util.Objects; +import java.util.Set; import org.apache.ignite.network.ClusterNode; /** * Nodes-based job execution target. */ public class NodesJobTarget implements JobTarget { - private final Collection nodes; + private final Set nodes; - public NodesJobTarget(Collection nodes) { + public NodesJobTarget(Set nodes) { Objects.requireNonNull(nodes); if (nodes.isEmpty()) { @@ -36,4 +36,8 @@ public NodesJobTarget(Collection nodes) { this.nodes = nodes; } + + public Set nodes() { + return nodes; + } } diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index b17f4f332a4..eae494d94ba 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -39,6 +39,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.ignite.compute.ColocatedExecutionTarget; import org.apache.ignite.compute.ComputeException; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.IgniteCompute; @@ -48,6 +49,7 @@ import org.apache.ignite.compute.JobStatus; import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.NodeNotFoundException; +import org.apache.ignite.compute.NodesJobTarget; import org.apache.ignite.compute.task.MapReduceJob; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.hlc.HybridClock; @@ -97,12 +99,30 @@ public IgniteComputeImpl(PlacementDriver placementDriver, TopologyService topolo @Override public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { - return null; + if (target instanceof NodesJobTarget) { + return submit(((NodesJobTarget) target).nodes(), descriptor, args); + } else if (target instanceof ColocatedExecutionTarget) { + ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; + var mapper = (Mapper) colocatedTarget.keyMapper(); + + if (mapper != null) { + return submitColocated( + colocatedTarget.tableName(), + colocatedTarget.key(), + mapper, + descriptor, + args); + } else { + return submitColocated(colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor, args); + } + } else { + throw new IllegalArgumentException("Unsupported job target: " + target); + } } @Override public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { - return null; + return sync(this.executeAsync(target, descriptor, args)); } @Override @@ -157,7 +177,6 @@ public R execute(Set nodes, JobDescriptor descriptor, Object... return sync(this.executeAsync(nodes, descriptor, args)); } - private static ClusterNode randomNode(Set nodes) { int nodesToSkip = ThreadLocalRandom.current().nextInt(nodes.size()); From 370583d4a230db97558abaa12b15ad85ab5deb1a Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:05:15 +0300 Subject: [PATCH 07/52] Fix AntiHijackIgniteCompute --- .../compute/AntiHijackIgniteCompute.java | 53 ++----------------- 1 file changed, 5 insertions(+), 48 deletions(-) diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/AntiHijackIgniteCompute.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/AntiHijackIgniteCompute.java index dadccee0356..770915221e4 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/AntiHijackIgniteCompute.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/AntiHijackIgniteCompute.java @@ -28,12 +28,11 @@ import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.compute.task.AntiHijackTaskExecution; import org.apache.ignite.internal.wrapper.Wrapper; import org.apache.ignite.network.ClusterNode; -import org.apache.ignite.table.Tuple; -import org.apache.ignite.table.mapper.Mapper; /** * Wrapper around {@link IgniteCompute} that adds protection against thread hijacking by users. @@ -51,55 +50,13 @@ public AntiHijackIgniteCompute(IgniteCompute compute, Executor asyncContinuation } @Override - public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { - return preventThreadHijack(compute.submit(nodes, descriptor, args)); + public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { + return preventThreadHijack(compute.submit(target, descriptor, args)); } @Override - public R execute(Set nodes, JobDescriptor descriptor, Object... args) { - return compute.execute(nodes, descriptor, args); - } - - @Override - public JobExecution submitColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return preventThreadHijack(compute.submitColocated(tableName, key, descriptor, args)); - } - - @Override - public JobExecution submitColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return preventThreadHijack(compute.submitColocated(tableName, key, keyMapper, descriptor, args)); - } - - @Override - public R executeColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return compute.executeColocated(tableName, key, descriptor, args); - } - - @Override - public R executeColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return compute.executeColocated(tableName, key, keyMapper, descriptor, args); + public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { + return compute.execute(target, descriptor, args); } @Override From 05cd0b1f8da53de8f66f2f98ac36498365f6f2a2 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:07:06 +0300 Subject: [PATCH 08/52] Fix FakeCompute --- .../ignite/client/fakes/FakeCompute.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java index 0fa44c70073..b9e0830b5f2 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java +++ b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java @@ -39,6 +39,7 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; import org.apache.ignite.Ignite; +import org.apache.ignite.compute.ColocatedExecutionTarget; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.IgniteCompute; @@ -47,6 +48,8 @@ import org.apache.ignite.compute.JobExecutionOptions; import org.apache.ignite.compute.JobState; import org.apache.ignite.compute.JobStatus; +import org.apache.ignite.compute.JobTarget; +import org.apache.ignite.compute.NodesJobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.compute.ComputeUtils; import org.apache.ignite.internal.compute.IgniteComputeInternal; @@ -126,6 +129,34 @@ public CompletableFuture> submitColocatedInternal(TableViewI return completedFuture(jobExecution(future != null ? future : completedFuture((R) nodeName))); } + @Override + public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { + if (target instanceof NodesJobTarget) { + return submit(((NodesJobTarget) target).nodes(), descriptor, args); + } else if (target instanceof ColocatedExecutionTarget) { + ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; + var mapper = (Mapper) colocatedTarget.keyMapper(); + + if (mapper != null) { + return submitColocated( + colocatedTarget.tableName(), + colocatedTarget.key(), + mapper, + descriptor, + args); + } else { + return submitColocated(colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor, args); + } + } else { + throw new IllegalArgumentException("Unsupported job target: " + target); + } + } + + @Override + public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { + return sync(this.executeAsync(target, descriptor, args)); + } + @Override public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); From 55887697613a947bb48450ea1e766d3d0ed6183c Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:08:01 +0300 Subject: [PATCH 09/52] Fix ClientCompute --- .../client/compute/ClientCompute.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index 130652d36ab..c55089a6414 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -35,11 +35,14 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; +import org.apache.ignite.compute.ColocatedExecutionTarget; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionOptions; +import org.apache.ignite.compute.JobTarget; +import org.apache.ignite.compute.NodesJobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.client.ClientUtils; import org.apache.ignite.internal.client.PayloadInputChannel; @@ -86,6 +89,34 @@ public ClientCompute(ReliableChannel ch, ClientTables tables) { this.tables = tables; } + @Override + public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { + if (target instanceof NodesJobTarget) { + return submit(((NodesJobTarget) target).nodes(), descriptor, args); + } else if (target instanceof ColocatedExecutionTarget) { + ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; + var mapper = (Mapper) colocatedTarget.keyMapper(); + + if (mapper != null) { + return submitColocated( + colocatedTarget.tableName(), + colocatedTarget.key(), + mapper, + descriptor, + args); + } else { + return submitColocated(colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor, args); + } + } else { + throw new IllegalArgumentException("Unsupported job target: " + target); + } + } + + @Override + public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { + return sync(this.submit(target, descriptor, args).resultAsync()); + } + @Override public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { Objects.requireNonNull(nodes); From fbc853037890594c8dd48add8c3875620e84262b Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:43:06 +0300 Subject: [PATCH 10/52] wip ClientCompute --- .../client/compute/ClientCompute.java | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index c55089a6414..b0768654b14 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -91,25 +91,47 @@ public ClientCompute(ReliableChannel ch, ClientTables tables) { @Override public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { + Objects.requireNonNull(target); + Objects.requireNonNull(descriptor); + if (target instanceof NodesJobTarget) { - return submit(((NodesJobTarget) target).nodes(), descriptor, args); - } else if (target instanceof ColocatedExecutionTarget) { + return new ClientJobExecution<>( + ch, + executeOnNodesAsync(( + (NodesJobTarget) target).nodes(), + descriptor.units(), + descriptor.jobClassName(), + descriptor.options(), + args)); + } + + if (target instanceof ColocatedExecutionTarget) { ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; var mapper = (Mapper) colocatedTarget.keyMapper(); if (mapper != null) { - return submitColocated( + return new ClientJobExecution<>(ch, doExecuteColocatedAsync( colocatedTarget.tableName(), colocatedTarget.key(), mapper, - descriptor, - args); + descriptor.units(), + descriptor.jobClassName(), + descriptor.options(), + args)); } else { - return submitColocated(colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor, args); + return new ClientJobExecution<>( + ch, + doExecuteColocatedAsync( + colocatedTarget.tableName(), + (Tuple) colocatedTarget.key(), + descriptor.units(), + descriptor.jobClassName(), + descriptor.options(), + args)); } - } else { - throw new IllegalArgumentException("Unsupported job target: " + target); } + + throw new IllegalArgumentException("Unsupported job target: " + target); } @Override From 1952fc728ba0b67bfcde3db503febd384e791a93 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:44:20 +0300 Subject: [PATCH 11/52] ClientCompute done --- .../client/compute/ClientCompute.java | 77 ------------------- 1 file changed, 77 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index b0768654b14..6c0dded325f 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -139,60 +139,6 @@ public R execute(JobTarget target, JobDescriptor descriptor, Object... args) return sync(this.submit(target, descriptor, args).resultAsync()); } - @Override - public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { - Objects.requireNonNull(nodes); - Objects.requireNonNull(descriptor); - - if (nodes.isEmpty()) { - throw new IllegalArgumentException("nodes must not be empty."); - } - - return new ClientJobExecution<>( - ch, - executeOnNodesAsync(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args)); - } - - @Override - public R execute(Set nodes, JobDescriptor descriptor, Object... args) { - return sync(this.executeAsync(nodes, descriptor, args)); - } - - /** {@inheritDoc} */ - @Override - public JobExecution submitColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - Objects.requireNonNull(tableName); - Objects.requireNonNull(key); - Objects.requireNonNull(descriptor); - - return new ClientJobExecution<>( - ch, - doExecuteColocatedAsync(tableName, key, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args)); - } - - /** {@inheritDoc} */ - @Override - public JobExecution submitColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - Objects.requireNonNull(tableName); - Objects.requireNonNull(key); - Objects.requireNonNull(keyMapper); - Objects.requireNonNull(descriptor); - - return new ClientJobExecution<>(ch, doExecuteColocatedAsync( - tableName, key, keyMapper, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args)); - } - private CompletableFuture doExecuteColocatedAsync( String tableName, Tuple key, @@ -232,29 +178,6 @@ private CompletableFuture doExecuteColocatedAsync( .thenCompose(Function.identity()); } - /** {@inheritDoc} */ - @Override - public R executeColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return sync(this.executeColocatedAsync(tableName, key, descriptor, args)); - } - - /** {@inheritDoc} */ - @Override - public R executeColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return sync(executeColocatedAsync(tableName, key, keyMapper, descriptor, args)); - } - /** {@inheritDoc} */ @Override public Map> submitBroadcast( From b078ca59bd4a8c5feec273612c30bba818b2b315 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:47:04 +0300 Subject: [PATCH 12/52] wip IgniteComputeImpl --- .../internal/compute/IgniteComputeImpl.java | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index eae494d94ba..f3bc00fd55f 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -99,30 +99,46 @@ public IgniteComputeImpl(PlacementDriver placementDriver, TopologyService topolo @Override public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { + Objects.requireNonNull(target); + Objects.requireNonNull(descriptor); + if (target instanceof NodesJobTarget) { - return submit(((NodesJobTarget) target).nodes(), descriptor, args); - } else if (target instanceof ColocatedExecutionTarget) { + Set nodes = ((NodesJobTarget) target).nodes(); + + return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); + } + + if (target instanceof ColocatedExecutionTarget) { ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; var mapper = (Mapper) colocatedTarget.keyMapper(); if (mapper != null) { - return submitColocated( - colocatedTarget.tableName(), - colocatedTarget.key(), - mapper, - descriptor, - args); + String tableName = colocatedTarget.tableName(); + Object key = colocatedTarget.key(); + + CompletableFuture> jobFut = requiredTable(tableName) + .thenCompose(table -> primaryReplicaForPartitionByMappedKey(table, key, mapper) + .thenApply(primaryNode -> executeOnOneNodeWithFailover( + primaryNode, + new NextColocatedWorkerSelector<>(placementDriver, topologyService, clock, table, key, mapper), + descriptor.units(), + descriptor.jobClassName(), + descriptor.options(), + args + ))); + + return new JobExecutionFutureWrapper<>(jobFut); } else { return submitColocated(colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor, args); } - } else { - throw new IllegalArgumentException("Unsupported job target: " + target); } + + throw new IllegalArgumentException("Unsupported job target: " + target); } @Override public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { - return sync(this.executeAsync(target, descriptor, args)); + return sync(this.submit(target, descriptor, args).resultAsync()); } @Override From 5dc1789dcac7dffb09bb1d808af49c72b3407168 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:49:58 +0300 Subject: [PATCH 13/52] wip IgniteComputeImpl --- .../internal/compute/IgniteComputeImpl.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index f3bc00fd55f..e02b408582e 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -111,12 +111,12 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob if (target instanceof ColocatedExecutionTarget) { ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; var mapper = (Mapper) colocatedTarget.keyMapper(); + String tableName = colocatedTarget.tableName(); + Object key = colocatedTarget.key(); + CompletableFuture> jobFut; if (mapper != null) { - String tableName = colocatedTarget.tableName(); - Object key = colocatedTarget.key(); - - CompletableFuture> jobFut = requiredTable(tableName) + jobFut = requiredTable(tableName) .thenCompose(table -> primaryReplicaForPartitionByMappedKey(table, key, mapper) .thenApply(primaryNode -> executeOnOneNodeWithFailover( primaryNode, @@ -127,10 +127,18 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob args ))); - return new JobExecutionFutureWrapper<>(jobFut); } else { - return submitColocated(colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor, args); + jobFut = requiredTable(tableName) + .thenCompose(table -> submitColocatedInternal( + table, + (Tuple) key, + descriptor.units(), + descriptor.jobClassName(), + descriptor.options(), + args)); } + + return new JobExecutionFutureWrapper<>(jobFut); } throw new IllegalArgumentException("Unsupported job target: " + target); From a51ff680fae1a289747eed31127d00452f9c64bd Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:50:56 +0300 Subject: [PATCH 14/52] IgniteComputeImpl cleanup done --- .../internal/compute/IgniteComputeImpl.java | 80 ------------------- 1 file changed, 80 deletions(-) diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index e02b408582e..afbc4c727c7 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -149,18 +149,6 @@ public R execute(JobTarget target, JobDescriptor descriptor, Object... args) return sync(this.submit(target, descriptor, args).resultAsync()); } - @Override - public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { - Objects.requireNonNull(nodes); - Objects.requireNonNull(descriptor); - - if (nodes.isEmpty()) { - throw new IllegalArgumentException("nodes must not be empty."); - } - - return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); - } - @Override public JobExecution executeAsyncWithFailover( Set nodes, @@ -196,11 +184,6 @@ public JobExecution executeAsyncWithFailover( )); } - @Override - public R execute(Set nodes, JobDescriptor descriptor, Object... args) { - return sync(this.executeAsync(nodes, descriptor, args)); - } - private static ClusterNode randomNode(Set nodes) { int nodesToSkip = ThreadLocalRandom.current().nextInt(nodes.size()); @@ -249,69 +232,6 @@ private boolean isLocal(ClusterNode targetNode) { return targetNode.equals(topologyService.localMember()); } - @Override - public JobExecution submitColocated( - String tableName, - Tuple tuple, - JobDescriptor descriptor, - Object... args - ) { - Objects.requireNonNull(tableName); - Objects.requireNonNull(tuple); - Objects.requireNonNull(descriptor); - - return new JobExecutionFutureWrapper<>( - requiredTable(tableName) - .thenCompose(table -> submitColocatedInternal( - table, tuple, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args)) - ); - } - - @Override - public JobExecution submitColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - Objects.requireNonNull(tableName); - Objects.requireNonNull(key); - Objects.requireNonNull(keyMapper); - Objects.requireNonNull(descriptor); - - return new JobExecutionFutureWrapper<>( - requiredTable(tableName) - .thenCompose(table -> primaryReplicaForPartitionByMappedKey(table, key, keyMapper) - .thenApply(primaryNode -> executeOnOneNodeWithFailover( - primaryNode, - new NextColocatedWorkerSelector<>(placementDriver, topologyService, clock, table, key, keyMapper), - descriptor.units(), descriptor.jobClassName(), descriptor.options(), args - ))) - ); - } - - @Override - public R executeColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return sync(this.executeColocatedAsync(tableName, key, descriptor, args)); - } - - @Override - public R executeColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return sync(executeColocatedAsync(tableName, key, keyMapper, descriptor, args)); - } - @Override public CompletableFuture> submitColocatedInternal( TableViewInternal table, From 88c3aaca6af00cab9162034ab5d68bfbbc75d0a8 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:59:06 +0300 Subject: [PATCH 15/52] removing old methods --- .../apache/ignite/compute/IgniteCompute.java | 37 +------------- .../org/apache/ignite/compute/JobTarget.java | 12 +++++ .../app/client/ItThinClientComputeTest.java | 50 ++++++++++--------- 3 files changed, 40 insertions(+), 59 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java index 75a9fe67b36..f8d7eb54ae3 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java @@ -97,41 +97,6 @@ R execute( - /** - * Submits a {@link ComputeJob} of the given class for an execution on a single node from a set of candidate nodes. - * - * @param Job result type. - * @param nodes Candidate nodes; the job will be executed on one of them. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @return Job execution object. - */ - default JobExecution submit( - Set nodes, - JobDescriptor descriptor, - Object... args - ) { - return submit(JobTarget.anyNode(nodes), descriptor, args); - } - - /** - * Submits a {@link ComputeJob} of the given class for an execution on a single node from a set of candidate nodes. A shortcut for - * {@code submit(...).resultAsync()}. - * - * @param Job result type. - * @param nodes Candidate nodes; the job will be executed on one of them. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @return Job result future. - */ - default CompletableFuture executeAsync( - Set nodes, - JobDescriptor descriptor, - Object... args - ) { - return this.submit(nodes, descriptor, args).resultAsync(); - } - /** * Executes a {@link ComputeJob} of the given class on a single node from a set of candidate nodes. * @@ -312,7 +277,7 @@ default CompletableFuture> executeBroadcastAsync( Object... args ) { Map> futures = nodes.stream() - .collect(toMap(identity(), node -> this.executeAsync(Set.of(node), descriptor, args))); + .collect(toMap(identity(), node -> this.executeAsync(JobTarget.node(node), descriptor, args))); return allOf(futures.values().toArray(CompletableFuture[]::new)) .thenApply(ignored -> { diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index e8439a3de3d..e0654f12429 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -17,6 +17,8 @@ package org.apache.ignite.compute; +import java.util.Collection; +import java.util.HashSet; import java.util.Set; import org.apache.ignite.network.ClusterNode; import org.apache.ignite.table.Tuple; @@ -46,6 +48,16 @@ static JobTarget anyNode(ClusterNode... nodes) { return new NodesJobTarget(Set.of(nodes)); } + /** + * Creates a job target for any node from the provided collection. + * + * @param nodes Collection of nodes. + * @return Job target. + */ + static JobTarget anyNode(Collection nodes) { + return new NodesJobTarget(new HashSet<>(nodes)); + } + /** * Creates a job target for any node from the provided collection. * diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java index 4bf18bc6df2..ad06e91042e 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java @@ -79,6 +79,7 @@ import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionContext; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.task.MapReduceJob; import org.apache.ignite.compute.task.MapReduceTask; import org.apache.ignite.compute.task.TaskExecution; @@ -129,8 +130,11 @@ void testExecuteOnSpecificNode() { @Test void testExecuteOnSpecificNodeAsync() { - JobExecution execution1 = client().compute().submit(Set.of(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); - JobExecution execution2 = client().compute().submit(Set.of(node(1)), JobDescriptor.builder(NodeNameJob.class).build()); + JobExecution execution1 = client().compute().submit( + JobTarget.node(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); + + JobExecution execution2 = client().compute().submit( + JobTarget.node(node(1)), JobDescriptor.builder(NodeNameJob.class).build()); assertThat(execution1.resultAsync(), willBe("itcct_n_3344")); assertThat(execution2.resultAsync(), willBe("itcct_n_3345")); @@ -141,7 +145,7 @@ void testExecuteOnSpecificNodeAsync() { @Test void testCancellingCompletedJob() { - JobExecution execution = client().compute().submit(Set.of(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); + JobExecution execution = client().compute().submit(JobTarget.node(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); assertThat(execution.resultAsync(), willBe("itcct_n_3344")); @@ -152,7 +156,7 @@ void testCancellingCompletedJob() { @Test void testChangingPriorityCompletedJob() { - JobExecution execution = client().compute().submit(Set.of(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); + JobExecution execution = client().compute().submit(JobTarget.node(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); assertThat(execution.resultAsync(), willBe("itcct_n_3344")); @@ -169,8 +173,8 @@ void testCancelOnSpecificNodeAsync(boolean asyncJob) { .builder(asyncJob ? AsyncSleepJob.class : SleepJob.class) .build(); - JobExecution execution1 = client().compute().submit(Set.of(node(0)), sleepJob, sleepMs); - JobExecution execution2 = client().compute().submit(Set.of(node(1)), sleepJob, sleepMs); + JobExecution execution1 = client().compute().submit(JobTarget.node(node(0)), sleepJob, sleepMs); + JobExecution execution2 = client().compute().submit(JobTarget.node(node(1)), sleepJob, sleepMs); await().until(execution1::statusAsync, willBe(jobStatusWithState(EXECUTING))); await().until(execution2::statusAsync, willBe(jobStatusWithState(EXECUTING))); @@ -186,18 +190,18 @@ void testCancelOnSpecificNodeAsync(boolean asyncJob) { void changeJobPriority() { int sleepMs = 1_000_000; JobDescriptor sleepJob = JobDescriptor.builder(SleepJob.class).build(); - Set nodes = Set.of(node(0)); + JobTarget target = JobTarget.node(node(0)); // Start 1 task in executor with 1 thread - JobExecution execution1 = client().compute().submit(nodes, sleepJob, sleepMs); + JobExecution execution1 = client().compute().submit(target, sleepJob, sleepMs); await().until(execution1::statusAsync, willBe(jobStatusWithState(EXECUTING))); // Start one more long lasting task - JobExecution execution2 = client().compute().submit(nodes, sleepJob, sleepMs); + JobExecution execution2 = client().compute().submit(target, sleepJob, sleepMs); await().until(execution2::statusAsync, willBe(jobStatusWithState(QUEUED))); // Start third task - JobExecution execution3 = client().compute().submit(nodes, sleepJob, sleepMs); + JobExecution execution3 = client().compute().submit(target, sleepJob, sleepMs); await().until(execution3::statusAsync, willBe(jobStatusWithState(QUEUED))); // Task 2 and 3 are not completed, in queue state @@ -230,7 +234,7 @@ void testExecuteOnRandomNode() { @Test void testExecuteOnRandomNodeAsync() { JobExecution execution = client().compute().submit( - new HashSet<>(sortedNodes()), JobDescriptor.builder(NodeNameJob.class).build()); + JobTarget.anyNode(sortedNodes()), JobDescriptor.builder(NodeNameJob.class).build()); assertThat( execution.resultAsync(), @@ -301,7 +305,7 @@ void testCancelBroadcastAllNodes() { @Test void testExecuteWithArgs() { JobExecution execution = client().compute().submit( - new HashSet<>(client().clusterNodes()), JobDescriptor.builder(ConcatJob.class).build(), 1, "2", 3.3); + JobTarget.anyNode(client().clusterNodes()), JobDescriptor.builder(ConcatJob.class).build(), 1, "2", 3.3); assertThat(execution.resultAsync(), willBe("1_2_3.3")); assertThat(execution.statusAsync(), willBe(jobStatusWithState(COMPLETED))); @@ -310,7 +314,7 @@ void testExecuteWithArgs() { @Test void testIgniteExceptionInJobPropagatesToClientWithMessageAndCodeAndTraceIdAsync() { IgniteException cause = getExceptionInJobExecutionAsync( - client().compute().submit(Set.of(node(0)), JobDescriptor.builder(IgniteExceptionJob.class).build()) + client().compute().submit(JobTarget.node(node(0)), JobDescriptor.builder(IgniteExceptionJob.class).build()) ); assertThat(cause.getMessage(), containsString("Custom job error")); @@ -323,7 +327,7 @@ void testIgniteExceptionInJobPropagatesToClientWithMessageAndCodeAndTraceIdAsync @Test void testIgniteExceptionInJobPropagatesToClientWithMessageAndCodeAndTraceIdSync() { IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().execute(Set.of(node(0)), JobDescriptor.builder(IgniteExceptionJob.class).build()) + () -> client().compute().execute(JobTarget.node(node(0)), JobDescriptor.builder(IgniteExceptionJob.class).build()) ); assertThat(cause.getMessage(), containsString("Custom job error")); @@ -337,7 +341,7 @@ void testIgniteExceptionInJobPropagatesToClientWithMessageAndCodeAndTraceIdSync( @ValueSource(booleans = {true, false}) void testExceptionInJobPropagatesToClientWithClassAndMessageAsync(boolean asyncJob) { IgniteException cause = getExceptionInJobExecutionAsync( - client().compute().submit(Set.of(node(0)), JobDescriptor.builder(ExceptionJob.class).build(), asyncJob) + client().compute().submit(JobTarget.node(node(0)), JobDescriptor.builder(ExceptionJob.class).build(), asyncJob) ); assertComputeExceptionWithClassAndMessage(cause); @@ -347,7 +351,7 @@ void testExceptionInJobPropagatesToClientWithClassAndMessageAsync(boolean asyncJ @ValueSource(booleans = {true, false}) void testExceptionInJobPropagatesToClientWithClassAndMessageSync(boolean asyncJob) { IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().execute(Set.of(node(0)), JobDescriptor.builder(ExceptionJob.class).build(), asyncJob) + () -> client().compute().execute(JobTarget.node(node(0)), JobDescriptor.builder(ExceptionJob.class).build(), asyncJob) ); assertComputeExceptionWithClassAndMessage(cause); @@ -357,7 +361,7 @@ void testExceptionInJobPropagatesToClientWithClassAndMessageSync(boolean asyncJo void testExceptionInJobWithSendServerExceptionStackTraceToClientPropagatesToClientWithStackTraceAsync() { // Second node has sendServerExceptionStackTraceToClient enabled. IgniteException cause = getExceptionInJobExecutionAsync( - client().compute().submit(Set.of(node(1)), JobDescriptor.builder(ExceptionJob.class) + client().compute().submit(JobTarget.node(node(1)), JobDescriptor.builder(ExceptionJob.class) .build()) ); @@ -368,7 +372,7 @@ void testExceptionInJobWithSendServerExceptionStackTraceToClientPropagatesToClie void testExceptionInJobWithSendServerExceptionStackTraceToClientPropagatesToClientWithStackTraceSync() { // Second node has sendServerExceptionStackTraceToClient enabled. IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().execute(Set.of(node(1)), JobDescriptor.builder(ExceptionJob.class).build()) + () -> client().compute().execute(JobTarget.node(node(1)), JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithStackTrace(cause); @@ -594,9 +598,9 @@ void testExecuteOnUnknownUnitWithLatestVersionThrows() { CompletionException.class, () -> { IgniteCompute igniteCompute = client().compute(); - Set nodes = Set.of(node(0)); + JobTarget target = JobTarget.node(node(0)); List units = List.of(new DeploymentUnit("u", "latest")); - igniteCompute.executeAsync(nodes, JobDescriptor.builder(NodeNameJob.class).units(units).build()).join(); + igniteCompute.executeAsync(target, JobDescriptor.builder(NodeNameJob.class).units(units).build()).join(); }); var cause = (IgniteException) ex.getCause(); @@ -630,7 +634,7 @@ void testDelayedJobExecutionThrowsWhenConnectionFails() throws Exception { try (IgniteClient client = builder.build()) { int delayMs = 3000; CompletableFuture jobFut = client.compute().executeAsync( - Set.of(node(0)), JobDescriptor.builder(SleepJob.class).build(), delayMs); + JobTarget.node(node(0)), JobDescriptor.builder(SleepJob.class).build(), delayMs); // Wait a bit and close the connection. Thread.sleep(10); @@ -678,7 +682,7 @@ void testExecuteColocatedEscapedTableName() { @ParameterizedTest @CsvSource({"1E3,-3", "1.12E5,-5", "1.12E5,0", "1.123456789,10", "1.123456789,5"}) void testBigDecimalPropagation(String number, int scale) { - BigDecimal res = client().compute().execute(Set.of(node(0)), JobDescriptor.builder(DecimalJob.class).build(), number, scale); + BigDecimal res = client().compute().execute(JobTarget.node(node(0)), JobDescriptor.builder(DecimalJob.class).build(), number, scale); var expected = new BigDecimal(number).setScale(scale, RoundingMode.HALF_UP); assertEquals(expected, res); @@ -725,7 +729,7 @@ void testExecuteMapReduceExceptionPropagation(Class taskClass) { } private void testEchoArg(Object arg) { - Object res = client().compute().execute(Set.of(node(0)), JobDescriptor.builder(EchoJob.class).build(), arg, arg.toString()); + Object res = client().compute().execute(JobTarget.node(node(0)), JobDescriptor.builder(EchoJob.class).build(), arg, arg.toString()); if (arg instanceof byte[]) { assertArrayEquals((byte[]) arg, (byte[]) res); From 99b922999e0db068706066abba382f6ffcf41e8b Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 14:59:49 +0300 Subject: [PATCH 16/52] removing old methods --- .../org/apache/ignite/internal/compute/IgniteComputeImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index afbc4c727c7..24efe43fbae 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -321,7 +321,7 @@ public R executeMapReduce(List units, String taskClassName, } private JobExecution submitJob(MapReduceJob runner) { - return submit(runner.nodes(), runner.jobDescriptor(), runner.args()); + return submit(JobTarget.anyNode(runner.nodes()), runner.jobDescriptor(), runner.args()); } @Override From 5f9e4fd3649ea1337f4eeb4e1b48fcab92574cd8 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 15:03:34 +0300 Subject: [PATCH 17/52] fixing tests --- .../internal/compute/ItComputeBaseTest.java | 17 +++++++------ .../compute/ItComputeTestEmbedded.java | 25 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java index 9d11f5bca8a..58f5242248b 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java @@ -48,6 +48,7 @@ import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; import org.apache.ignite.internal.app.IgniteImpl; @@ -96,7 +97,7 @@ void executesWrongJobClassLocallyAsync(String jobClassName, int errorCode, Strin IgniteImpl entryNode = node(0); ExecutionException ex = assertThrows(ExecutionException.class, () -> entryNode.compute().executeAsync( - Set.of(entryNode.node()), JobDescriptor.builder(jobClassName).units(units()).build()) + JobTarget.node(entryNode.node()), JobDescriptor.builder(jobClassName).units(units()).build()) .get(1, TimeUnit.SECONDS)); assertTraceableException(ex, ComputeException.class, errorCode, msg); @@ -120,7 +121,7 @@ void executesWrongJobClassOnRemoteNodesAsync(String jobClassName, int errorCode, Ignite entryNode = node(0); ExecutionException ex = assertThrows(ExecutionException.class, () -> entryNode.compute().executeAsync( - Set.of(node(1).node(), node(2).node()), JobDescriptor.builder(jobClassName).units(units()).build()) + JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(jobClassName).units(units()).build()) .get(1, TimeUnit.SECONDS)); assertTraceableException(ex, ComputeException.class, errorCode, msg); @@ -143,7 +144,7 @@ void executesJobLocallyAsync() { IgniteImpl entryNode = node(0); JobExecution execution = entryNode.compute().submit( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(concatJobClassName()).units(units()).build(), "a", 42); @@ -169,7 +170,7 @@ void executesJobOnRemoteNodesAsync() { IgniteImpl entryNode = node(0); JobExecution execution = entryNode.compute().submit( - Set.of(node(1).node(), node(2).node()), + JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(concatJobClassName()).units(units()).build(), new Object[]{"a", 42}); @@ -183,7 +184,7 @@ void localExecutionActuallyUsesLocalNode() { IgniteImpl entryNode = node(0); CompletableFuture fut = entryNode.compute().executeAsync( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); assertThat(fut, willBe(entryNode.name())); @@ -195,7 +196,7 @@ void remoteExecutionActuallyUsesRemoteNode() { IgniteImpl remoteNode = node(1); CompletableFuture fut = entryNode.compute().executeAsync( - Set.of(remoteNode.node()), + JobTarget.node(remoteNode.node()), JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); assertThat(fut, willBe(remoteNode.name())); @@ -217,7 +218,7 @@ void executesFailingJobLocallyAsync() { IgniteImpl entryNode = node(0); JobExecution execution = entryNode.compute().submit( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(failingJobClassName()).units(units()).build()); ExecutionException ex = assertThrows(ExecutionException.class, () -> execution.resultAsync().get(1, TimeUnit.SECONDS)); @@ -244,7 +245,7 @@ void executesFailingJobOnRemoteNodesAsync() { Ignite entryNode = node(0); JobExecution execution = entryNode.compute().submit( - Set.of(node(1).node(), node(2).node()), + JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(failingJobClassName()).units(units()).build()); ExecutionException ex = assertThrows(ExecutionException.class, () -> execution.resultAsync().get(1, TimeUnit.SECONDS)); diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java index 4aed1cf5253..1b0e9a07fea 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java @@ -53,6 +53,7 @@ import org.apache.ignite.compute.JobExecutionContext; import org.apache.ignite.compute.JobExecutionOptions; import org.apache.ignite.compute.JobState; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.lang.IgniteInternalCheckedException; import org.apache.ignite.internal.lang.IgniteInternalException; @@ -83,7 +84,7 @@ void cancelsJobLocally() { IgniteImpl entryNode = node(0); JobDescriptor job = JobDescriptor.builder(WaitLatchJob.class).units(units()).build(); - JobExecution execution = entryNode.compute().submit(Set.of(entryNode.node()), job, new CountDownLatch(1)); + JobExecution execution = entryNode.compute().submit(JobTarget.node(entryNode.node()), job, new CountDownLatch(1)); await().until(execution::statusAsync, willBe(jobStatusWithState(JobState.EXECUTING))); @@ -97,7 +98,7 @@ void cancelsJobRemotely() { IgniteImpl entryNode = node(0); JobDescriptor job = JobDescriptor.builder(WaitLatchJob.class).units(units()).build(); - JobExecution execution = entryNode.compute().submit(Set.of(node(1).node()), job, new CountDownLatch(1)); + JobExecution execution = entryNode.compute().submit(JobTarget.node(node(1).node()), job, new CountDownLatch(1)); await().until(execution::statusAsync, willBe(jobStatusWithState(JobState.EXECUTING))); @@ -111,7 +112,7 @@ void changeExecutingJobPriorityLocally() { IgniteImpl entryNode = node(0); JobDescriptor job = JobDescriptor.builder(WaitLatchJob.class).units(units()).build(); - JobExecution execution = entryNode.compute().submit(Set.of(entryNode.node()), job, new CountDownLatch(1)); + JobExecution execution = entryNode.compute().submit(JobTarget.node(entryNode.node()), job, new CountDownLatch(1)); await().until(execution::statusAsync, willBe(jobStatusWithState(JobState.EXECUTING))); assertThat(execution.changePriorityAsync(2), willBe(false)); @@ -123,7 +124,7 @@ void changeExecutingJobPriorityRemotely() { IgniteImpl entryNode = node(0); JobDescriptor job = JobDescriptor.builder(WaitLatchJob.class).units(units()).build(); - JobExecution execution = entryNode.compute().submit(Set.of(node(1).node()), job, new CountDownLatch(1)); + JobExecution execution = entryNode.compute().submit(JobTarget.node(node(1).node()), job, new CountDownLatch(1)); await().until(execution::statusAsync, willBe(jobStatusWithState(JobState.EXECUTING))); assertThat(execution.changePriorityAsync(2), willBe(false)); @@ -133,21 +134,21 @@ void changeExecutingJobPriorityRemotely() { @Test void changeJobPriorityLocally() { IgniteImpl entryNode = node(0); - Set nodes = Set.of(entryNode.node()); + JobTarget jobTarget = JobTarget.node(entryNode.node()); CountDownLatch countDownLatch = new CountDownLatch(1); JobDescriptor job = JobDescriptor.builder(WaitLatchJob.class).units(units()).build(); // Start 1 task in executor with 1 thread - JobExecution execution1 = entryNode.compute().submit(nodes, job, countDownLatch); + JobExecution execution1 = entryNode.compute().submit(jobTarget, job, countDownLatch); await().until(execution1::statusAsync, willBe(jobStatusWithState(JobState.EXECUTING))); // Start one more task - JobExecution execution2 = entryNode.compute().submit(nodes, job, new CountDownLatch(1)); + JobExecution execution2 = entryNode.compute().submit(jobTarget, job, new CountDownLatch(1)); await().until(execution2::statusAsync, willBe(jobStatusWithState(JobState.QUEUED))); // Start third task - JobExecution execution3 = entryNode.compute().submit(nodes, job, countDownLatch); + JobExecution execution3 = entryNode.compute().submit(jobTarget, job, countDownLatch); await().until(execution3::statusAsync, willBe(jobStatusWithState(JobState.QUEUED))); // Task 1 and 2 are not competed, in queue state @@ -174,23 +175,23 @@ void changeJobPriorityLocally() { @Test void executesJobLocallyWithOptions() { IgniteImpl entryNode = node(0); - Set nodes = Set.of(entryNode.node()); + JobTarget jobTarget = JobTarget.node(entryNode.node()); CountDownLatch countDownLatch = new CountDownLatch(1); JobDescriptor job = JobDescriptor.builder(WaitLatchJob.class).units(units()).build(); // Start 1 task in executor with 1 thread - JobExecution execution1 = entryNode.compute().submit(nodes, job, new Object[]{countDownLatch}); + JobExecution execution1 = entryNode.compute().submit(jobTarget, job, new Object[]{countDownLatch}); await().until(execution1::statusAsync, willBe(jobStatusWithState(JobState.EXECUTING))); // Start one more task - JobExecution execution2 = entryNode.compute().submit(nodes, job, new Object[]{new CountDownLatch(1)}); + JobExecution execution2 = entryNode.compute().submit(jobTarget, job, new Object[]{new CountDownLatch(1)}); await().until(execution2::statusAsync, willBe(jobStatusWithState(JobState.QUEUED))); // Start third task it should be before task2 in the queue due to higher priority in options JobExecutionOptions options = JobExecutionOptions.builder().priority(1).maxRetries(2).build(); JobExecution execution3 = entryNode.compute().submit( - nodes, + jobTarget, JobDescriptor.builder(WaitLatchThrowExceptionOnFirstExecutionJob.class) .units(units()) .options(options) From f1ff5734be98fb54eb6b8aea0ad1d3210dfdadc7 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 15:06:23 +0300 Subject: [PATCH 18/52] fixing tests --- .../internal/compute/ItComputeErrorsBaseTest.java | 4 +++- .../internal/compute/ItComputeTestStandalone.java | 14 +++++++------- .../internal/compute/ItExecutionsCleanerTest.java | 4 +++- .../compute/ItFailoverCandidateNotFoundTest.java | 4 +++- .../internal/compute/ItWorkerShutdownTest.java | 5 +++-- .../threading/ItComputeApiThreadingTest.java | 5 +++-- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java index b9e999b6830..232db5333ab 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.NodeNotFoundException; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; import org.apache.ignite.internal.compute.utils.InteractiveJobs; @@ -138,6 +139,7 @@ void broadcastAsync() { protected abstract IgniteCompute compute(); private TestingJobExecution executeGlobalInteractiveJob(Set nodes) { - return new TestingJobExecution<>(compute().submit(nodes, JobDescriptor.builder(InteractiveJobs.globalJob().name()).build())); + return new TestingJobExecution<>( + compute().submit(JobTarget.anyNode(nodes), JobDescriptor.builder(InteractiveJobs.globalJob().name()).build())); } } diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestStandalone.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestStandalone.java index 546e73c820d..3d500298d06 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestStandalone.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestStandalone.java @@ -31,11 +31,11 @@ import java.io.InputStream; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.JobDescriptor; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.version.Version; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.deployunit.NodesToDeploy; @@ -105,7 +105,7 @@ void executesJobWithNonExistingUnit() { List nonExistingUnits = List.of(new DeploymentUnit("non-existing", "1.0.0")); CompletableFuture result = entryNode.compute().executeAsync( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(concatJobClassName()).units(nonExistingUnits).build(), "a", 42); @@ -128,13 +128,13 @@ void executesJobWithLatestUnitVersion() throws IOException { deployJar(entryNode, firstVersion.name(), firstVersion.version(), "ignite-unit-test-job1-1.0-SNAPSHOT.jar"); JobDescriptor job = JobDescriptor.builder("org.apache.ignite.internal.compute.UnitJob").units(jobUnits).build(); - CompletableFuture result1 = entryNode.compute().executeAsync(Set.of(entryNode.node()), job); + CompletableFuture result1 = entryNode.compute().executeAsync(JobTarget.node(entryNode.node()), job); assertThat(result1, willBe(1)); DeploymentUnit secondVersion = new DeploymentUnit("latest-unit", Version.parseVersion("1.0.1")); deployJar(entryNode, secondVersion.name(), secondVersion.version(), "ignite-unit-test-job2-1.0-SNAPSHOT.jar"); - CompletableFuture result2 = entryNode.compute().executeAsync(Set.of(entryNode.node()), job); + CompletableFuture result2 = entryNode.compute().executeAsync(JobTarget.node(entryNode.node()), job); assertThat(result2, willBe("Hello World!")); } @@ -143,7 +143,7 @@ void undeployAcquiredUnit() { IgniteImpl entryNode = node(0); CompletableFuture job = entryNode.compute().executeAsync( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(SleepJob.class).units(units).build(), 3L); @@ -168,11 +168,11 @@ void executeJobWithObsoleteUnit() { IgniteImpl entryNode = node(0); JobDescriptor job = JobDescriptor.builder(SleepJob.class).units(units).build(); - CompletableFuture successJob = entryNode.compute().executeAsync(Set.of(entryNode.node()), job, 2L); + CompletableFuture successJob = entryNode.compute().executeAsync(JobTarget.node(entryNode.node()), job, 2L); assertThat(entryNode.deployment().undeployAsync(unit.name(), unit.version()), willCompleteSuccessfully()); - CompletableFuture failedJob = entryNode.compute().executeAsync(Set.of(entryNode.node()), job, 2L); + CompletableFuture failedJob = entryNode.compute().executeAsync(JobTarget.node(entryNode.node()), job, 2L); CompletionException ex0 = assertThrows(CompletionException.class, failedJob::join); assertComputeException( diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItExecutionsCleanerTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItExecutionsCleanerTest.java index 2343213f83b..e4e66b2604b 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItExecutionsCleanerTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItExecutionsCleanerTest.java @@ -30,6 +30,7 @@ import org.apache.ignite.compute.ComputeException; import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.compute.utils.InteractiveJobs; @@ -245,6 +246,7 @@ void failover() throws Exception { private static TestingJobExecution submit(Set nodes) { IgniteCompute igniteCompute = CLUSTER.node(0).compute(); - return new TestingJobExecution<>(igniteCompute.submit(nodes, JobDescriptor.builder(InteractiveJobs.globalJob().name()).build())); + return new TestingJobExecution<>( + igniteCompute.submit(JobTarget.anyNode(nodes), JobDescriptor.builder(InteractiveJobs.globalJob().name()).build())); } } diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItFailoverCandidateNotFoundTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItFailoverCandidateNotFoundTest.java index 92b3b468bf4..e1cd0e1ce67 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItFailoverCandidateNotFoundTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItFailoverCandidateNotFoundTest.java @@ -26,6 +26,7 @@ import org.apache.ignite.client.IgniteClient; import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.ClusterPerTestIntegrationTest; import org.apache.ignite.internal.compute.utils.InteractiveJobs; import org.apache.ignite.internal.compute.utils.TestingJobExecution; @@ -95,6 +96,7 @@ private void failoverCandidateLeavesCluster(IgniteCompute compute) throws Except } private static TestingJobExecution executeGlobalInteractiveJob(IgniteCompute compute, Set nodes) { - return new TestingJobExecution<>(compute.submit(nodes, JobDescriptor.builder(InteractiveJobs.globalJob().name()).build())); + return new TestingJobExecution<>( + compute.submit(JobTarget.anyNode(nodes), JobDescriptor.builder(InteractiveJobs.globalJob().name()).build())); } } diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java index 69b3851fa1a..758782a1b0e 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java @@ -39,6 +39,7 @@ import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.ClusterPerTestIntegrationTest; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.compute.utils.InteractiveJobs; @@ -363,7 +364,7 @@ private IgniteImpl nodeByName(String candidateName) { private TestingJobExecution executeGlobalInteractiveJob(IgniteImpl entryNode, Set nodes) { return new TestingJobExecution<>( compute(entryNode).submit( - clusterNodesByNames(nodes), + JobTarget.anyNode(clusterNodesByNames(nodes)), JobDescriptor.builder(InteractiveJobs.globalJob().name()).build()) ); } @@ -378,7 +379,7 @@ private void createReplicatedTestTableWithOneRow() { executeSql("INSERT INTO test(k, v) VALUES (1, 101)"); } - private List allNodeNames() { + private static List allNodeNames() { return new ArrayList<>(NODES_NAMES_TO_INDEXES.keySet()); } } diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java index d19788588a9..a377163fb42 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionContext; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; import org.apache.ignite.internal.compute.IgniteComputeImpl; import org.apache.ignite.internal.wrapper.Wrappers; @@ -148,7 +149,7 @@ public CompletableFuture executeAsync(JobExecutionContext context, Objec } private enum ComputeAsyncOperation { - EXECUTE_ASYNC(compute -> compute.executeAsync(justNonEntryNode(), JobDescriptor.builder(NoOpJob.class).build())), + EXECUTE_ASYNC(compute -> compute.executeAsync(JobTarget.anyNode(justNonEntryNode()), JobDescriptor.builder(NoOpJob.class).build())), EXECUTE_COLOCATED_BY_TUPLE_ASYNC(compute -> compute.executeColocatedAsync(TABLE_NAME, KEY_TUPLE, JobDescriptor.builder(NoOpJob.class).build())), EXECUTE_COLOCATED_BY_KEY_ASYNC(compute -> @@ -167,7 +168,7 @@ CompletableFuture executeOn(IgniteCompute compute) { } private enum ComputeSubmitOperation { - SUBMIT(compute -> compute.submit(justNonEntryNode(), JobDescriptor.builder(NoOpJob.class).build())), + SUBMIT(compute -> compute.submit(JobTarget.anyNode(justNonEntryNode()), JobDescriptor.builder(NoOpJob.class).build())), SUBMIT_COLOCATED_BY_TUPLE(compute -> compute.submitColocated(TABLE_NAME, KEY_TUPLE, JobDescriptor.builder(NoOpJob.class).build())), SUBMIT_COLOCATED_BY_KEY(compute -> compute.submitColocated( TABLE_NAME, KEY, Mapper.of(Integer.class), JobDescriptor.builder(NoOpJob.class).build()) From a7c0a59dbbf58daa6d8c17e12ca13f9aac205a7a Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 15:12:30 +0300 Subject: [PATCH 19/52] Remove all old methods - compilation broken --- .../apache/ignite/compute/IgniteCompute.java | 162 +----------------- 1 file changed, 1 insertion(+), 161 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java index f8d7eb54ae3..9db34bbe1be 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/IgniteCompute.java @@ -30,8 +30,6 @@ import org.apache.ignite.compute.task.MapReduceTask; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.network.ClusterNode; -import org.apache.ignite.table.Tuple; -import org.apache.ignite.table.mapper.Mapper; /** * Provides the ability to execute Compute jobs. @@ -89,164 +87,6 @@ R execute( Object... args ); - - - - /** OLD METHODS BELOW */ - - - - - /** - * Executes a {@link ComputeJob} of the given class on a single node from a set of candidate nodes. - * - * @param Job result type - * @param nodes Candidate nodes; the job will be executed on one of them. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @return Job result. - * @throws ComputeException If there is any problem executing the job. - */ - default R execute( - Set nodes, - JobDescriptor descriptor, - Object... args - ) { - return execute(JobTarget.anyNode(nodes), descriptor, args); - } - - /** - * Submits a job of the given class for the execution on the node where the given key is located. The node is a leader of the - * corresponding RAFT group. - * - * @param tableName Name of the table whose key is used to determine the node to execute the job on. - * @param key Key that identifies the node to execute the job on. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @param Job result type. - * @return Job execution object. - */ - default JobExecution submitColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return submit(JobTarget.colocated(tableName, key), descriptor, args); - } - - /** - * Submits a job of the given class for the execution on the node where the given key is located. The node is a leader of the - * corresponding RAFT group. - * - * @param tableName Name of the table whose key is used to determine the node to execute the job on. - * @param key Key that identifies the node to execute the job on. - * @param keyMapper Mapper used to map the key to a binary representation. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @param Job result type. - * @return Job execution object. - */ - default JobExecution submitColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return submit(JobTarget.colocated(tableName, key, keyMapper), descriptor, args); - } - - /** - * Submits a job of the given class for the execution on the node where the given key is located. The node is a leader of the - * corresponding RAFT group. A shortcut for {@code submitColocated(...).resultAsync()}. - * - * @param tableName Name of the table whose key is used to determine the node to execute the job on. - * @param key Key that identifies the node to execute the job on. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @param Job result type. - * @return Job result future. - */ - default CompletableFuture executeColocatedAsync( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return this.submitColocated(tableName, key, descriptor, args).resultAsync(); - } - - /** - * Submits a job of the given class for the execution on the node where the given key is located. The node is a leader of the - * corresponding RAFT group. A shortcut for {@code submitColocated(...).resultAsync()}. - * - * @param tableName Name of the table whose key is used to determine the node to execute the job on. - * @param key Key that identifies the node to execute the job on. - * @param keyMapper Mapper used to map the key to a binary representation. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @param Job result type. - * @return Job result future. - */ - default CompletableFuture executeColocatedAsync( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return this.submitColocated(tableName, key, keyMapper, descriptor, args).resultAsync(); - } - - /** - * Executes a job of the given class on the node where the given key is located. The node is a leader of the corresponding RAFT group. - * - * @param Job result type. - * @param tableName Name of the table whose key is used to determine the node to execute the job on. - * @param key Key that identifies the node to execute the job on. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @return Job result. - * @throws ComputeException If there is any problem executing the job. - */ - default R executeColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args) { - return execute(JobTarget.colocated(tableName, key), descriptor, args); - } - - /** - * Executes a job of the given class on the node where the given key is located. The node is a leader of the corresponding RAFT group. - * - * @param Job result type. - * @param tableName Name of the table whose key is used to determine the node to execute the job on. - * @param key Key that identifies the node to execute the job on. - * @param keyMapper Mapper used to map the key to a binary representation. - * @param descriptor Job descriptor. - * @param args Arguments of the job. - * @return Job result. - * @throws ComputeException If there is any problem executing the job. - */ - default R executeColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args) { - return execute(JobTarget.colocated(tableName, key, keyMapper), descriptor, args); - } - - - - - /** BROADCAST METHODS BELOW - NOT AFFECTED BY ExecutionTarget */ - - - - /** * Submits a {@link ComputeJob} of the given class for an execution on all nodes in the given node set. * @@ -310,7 +150,7 @@ default Map executeBroadcast( Map map = new HashMap<>(); for (ClusterNode node : nodes) { - map.put(node, execute(Set.of(node), descriptor, args)); + map.put(node, execute(JobTarget.node(node), descriptor, args)); } return map; From 4105a713519547f79b910ebccb0acee95afa2182 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 18 Jun 2024 15:15:05 +0300 Subject: [PATCH 20/52] Remove all old methods - compilation broken - fixing tests --- .../internal/compute/ItComputeBaseTest.java | 47 ++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java index 58f5242248b..0f39ff53763 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java @@ -86,7 +86,7 @@ void executesWrongJobClassLocally(String jobClassName, int errorCode, String msg IgniteImpl entryNode = node(0); IgniteException ex = assertThrows(IgniteException.class, () -> - entryNode.compute().execute(Set.of(entryNode.node()), JobDescriptor.builder(jobClassName).units(units()).build())); + entryNode.compute().execute(JobTarget.node(entryNode.node()), JobDescriptor.builder(jobClassName).units(units()).build())); assertTraceableException(ex, ComputeException.class, errorCode, msg); } @@ -109,7 +109,7 @@ void executesWrongJobClassOnRemoteNodes(String jobClassName, int errorCode, Stri Ignite entryNode = node(0); IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute( - Set.of(node(1).node(), node(2).node()), + JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(jobClassName).units(units()).build())); assertTraceableException(ex, ComputeException.class, errorCode, msg); @@ -132,7 +132,7 @@ void executesJobLocally() { IgniteImpl entryNode = node(0); String result = entryNode.compute().execute( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(concatJobClassName()).units(units()).build(), "a", 42); @@ -158,7 +158,7 @@ void executesJobOnRemoteNodes() { Ignite entryNode = node(0); String result = entryNode.compute().execute( - Set.of(node(1).node(), node(2).node()), + JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(concatJobClassName()).units(units()).build(), "a", 42); @@ -207,7 +207,7 @@ void executesFailingJobLocally() { IgniteImpl entryNode = node(0); IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(failingJobClassName()).units(units()).build())); assertComputeException(ex, "JobException", "Oops"); @@ -234,7 +234,7 @@ void executesFailingJobOnRemoteNodes() { Ignite entryNode = node(0); IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute( - Set.of(node(1).node(), node(2).node()), + JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(failingJobClassName()).units(units()).build())); assertComputeException(ex, "JobException", "Oops"); @@ -322,9 +322,8 @@ void executesColocatedWithTupleKey() { IgniteImpl entryNode = node(0); - String actualNodeName = entryNode.compute().executeColocated( - "test", - Tuple.create(Map.of("k", 1)), + String actualNodeName = entryNode.compute().execute( + JobTarget.colocated("test", Tuple.create(Map.of("k", 1))), JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); assertThat(actualNodeName, in(allNodeNames())); @@ -336,9 +335,8 @@ void executesColocatedWithTupleKeyAsync() { IgniteImpl entryNode = node(0); - JobExecution execution = entryNode.compute().submitColocated( - "test", - Tuple.create(Map.of("k", 1)), + JobExecution execution = entryNode.compute().submit( + JobTarget.colocated("test", Tuple.create(Map.of("k", 1))), JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); assertThat(execution.resultAsync(), willBe(in(allNodeNames()))); @@ -352,9 +350,8 @@ public void executesColocatedWithNonConsecutiveKeyColumnOrder() { sql("CREATE TABLE test (k int, key_int int, v int, key_str VARCHAR, CONSTRAINT PK PRIMARY KEY (key_int, key_str))"); sql("INSERT INTO test VALUES (1, 2, 3, '4')"); - String actualNodeName = node(0).compute().executeColocated( - "test", - Tuple.create(Map.of("key_int", 2, "key_str", "4")), + String actualNodeName = node(0).compute().execute( + JobTarget.colocated("test", Tuple.create(Map.of("key_int", 2, "key_str", "4"))), JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); assertThat(actualNodeName, in(allNodeNames())); @@ -365,12 +362,9 @@ void executeColocatedThrowsTableNotFoundExceptionWhenTableDoesNotExist() { IgniteImpl entryNode = node(0); var ex = assertThrows(CompletionException.class, - () -> { - entryNode.compute().submitColocated( - "\"bad-table\"", - Tuple.create(Map.of("k", 1)), - JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()).resultAsync().join(); - }); + () -> entryNode.compute().submit( + JobTarget.colocated("\"bad-table\"", Tuple.create(Map.of("k", 1))), + JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()).resultAsync().join()); assertInstanceOf(TableNotFoundException.class, ex.getCause()); assertThat(ex.getCause().getMessage(), containsString("The table does not exist [name=\"PUBLIC\".\"bad-table\"]")); @@ -395,8 +389,9 @@ void executesColocatedWithMappedKey() { IgniteImpl entryNode = node(0); - String actualNodeName = entryNode.compute().executeColocated( - "test", 1, Mapper.of(Integer.class), JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); + String actualNodeName = entryNode.compute().execute( + JobTarget.colocated("test", 1, Mapper.of(Integer.class)), + JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); assertThat(actualNodeName, in(allNodeNames())); } @@ -407,10 +402,8 @@ void executesColocatedWithMappedKeyAsync() { IgniteImpl entryNode = node(0); - JobExecution execution = entryNode.compute().submitColocated( - "test", - 1, - Mapper.of(Integer.class), + JobExecution execution = entryNode.compute().submit( + JobTarget.colocated("test", 1, Mapper.of(Integer.class)), JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build()); assertThat(execution.resultAsync(), willBe(in(allNodeNames()))); From 9f852517e84ed5b51b9181c962072c8bbd0f89ec Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 09:24:44 +0300 Subject: [PATCH 21/52] wip fixing tests --- .../app/client/ItThinClientComputeTest.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java index ad06e91042e..31ad423009e 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java @@ -121,8 +121,8 @@ void testClusterNodes() { @Test void testExecuteOnSpecificNode() { - String res1 = client().compute().execute(Set.of(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); - String res2 = client().compute().execute(Set.of(node(1)), JobDescriptor.builder(NodeNameJob.class).build()); + String res1 = client().compute().execute(JobTarget.node(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); + String res2 = client().compute().execute(JobTarget.node(node(1)), JobDescriptor.builder(NodeNameJob.class).build()); assertEquals("itcct_n_3344", res1); assertEquals("itcct_n_3345", res2); @@ -226,7 +226,7 @@ void changeJobPriority() { @Test void testExecuteOnRandomNode() { - String res = client().compute().execute(new HashSet<>(sortedNodes()), JobDescriptor.builder(NodeNameJob.class).build()); + String res = client().compute().execute(JobTarget.anyNode(sortedNodes()), JobDescriptor.builder(NodeNameJob.class).build()); assertTrue(Set.of("itcct_n_3344", "itcct_n_3345").contains(res)); } @@ -395,7 +395,7 @@ void testExceptionInColocatedTupleJobPropagatesToClientWithClassAndMessageAsync( var key = Tuple.create().set(COLUMN_KEY, 1); IgniteException cause = getExceptionInJobExecutionAsync( - client().compute().submitColocated(TABLE_NAME, key, JobDescriptor.builder(ExceptionJob.class).build())); + client().compute().submit(JobTarget.colocated(TABLE_NAME, key), JobDescriptor.builder(ExceptionJob.class).build())); assertComputeExceptionWithClassAndMessage(cause); } @@ -405,7 +405,7 @@ void testExceptionInColocatedTupleJobPropagatesToClientWithClassAndMessageSync() var key = Tuple.create().set(COLUMN_KEY, 1); IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().executeColocated(TABLE_NAME, key, JobDescriptor.builder(ExceptionJob.class).build()) + () -> client().compute().execute(JobTarget.colocated(TABLE_NAME, key), JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithClassAndMessage(cause); @@ -418,7 +418,7 @@ void testExceptionInColocatedTupleJobWithSendServerExceptionStackTraceToClientPr IgniteCompute igniteCompute = client().compute(); IgniteException cause = getExceptionInJobExecutionAsync( - igniteCompute.submitColocated(TABLE_NAME, key, JobDescriptor.builder(ExceptionJob.class).build()) + igniteCompute.submit(JobTarget.colocated(TABLE_NAME, key), JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithStackTrace(cause); @@ -430,7 +430,7 @@ void testExceptionInColocatedTupleJobWithSendServerExceptionStackTraceToClientPr var key = Tuple.create().set(COLUMN_KEY, 2); IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().executeColocated(TABLE_NAME, key, JobDescriptor.builder(ExceptionJob.class).build()) + () -> client().compute().execute(JobTarget.colocated(TABLE_NAME, key), JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithStackTrace(cause); @@ -443,7 +443,7 @@ void testExceptionInColocatedPojoJobPropagatesToClientWithClassAndMessageAsync() IgniteCompute igniteCompute = client().compute(); IgniteException cause = getExceptionInJobExecutionAsync( - igniteCompute.submitColocated(TABLE_NAME, key, mapper, JobDescriptor.builder(ExceptionJob.class) + igniteCompute.submit(JobTarget.colocated(TABLE_NAME, key, mapper), JobDescriptor.builder(ExceptionJob.class) .build()) ); @@ -456,7 +456,7 @@ void testExceptionInColocatedPojoJobPropagatesToClientWithClassAndMessageSync() Mapper mapper = Mapper.of(TestPojo.class); IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().executeColocated(TABLE_NAME, key, mapper, JobDescriptor.builder(ExceptionJob.class).build()) + () -> client().compute().execute(JobTarget.colocated(TABLE_NAME, key, mapper), JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithClassAndMessage(cause); @@ -470,7 +470,7 @@ void testExceptionInColocatedPojoJobWithSendServerExceptionStackTraceToClientPro IgniteCompute igniteCompute = client().compute(); IgniteException cause = getExceptionInJobExecutionAsync( - igniteCompute.submitColocated(TABLE_NAME, key, mapper, JobDescriptor.builder(ExceptionJob.class).build()) + igniteCompute.submit(JobTarget.colocated(TABLE_NAME, key, mapper), JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithStackTrace(cause); @@ -483,7 +483,8 @@ void testExceptionInColocatedPojoJobWithSendServerExceptionStackTraceToClientPro Mapper mapper = Mapper.of(TestPojo.class); IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().executeColocated(TABLE_NAME, key, mapper, JobDescriptor.builder(ExceptionJob.class).build()) + () -> client().compute().execute( + JobTarget.colocated(TABLE_NAME, key, mapper), JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithStackTrace(cause); From 2f7f520060a5c1be220416a31abdbadf9df9d995 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 09:27:02 +0300 Subject: [PATCH 22/52] wip fixing tests --- .../app/client/ItThinClientComputeTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java index 31ad423009e..993d131f4ac 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java @@ -533,8 +533,8 @@ void testExecuteColocatedTupleRunsComputeJobOnKeyNode(int key, int port) { var keyTuple = Tuple.create().set(COLUMN_KEY, key); IgniteCompute igniteCompute = client().compute(); - JobExecution tupleExecution = igniteCompute.submitColocated( - TABLE_NAME, keyTuple, JobDescriptor.builder(NodeNameJob.class).build()); + JobExecution tupleExecution = igniteCompute.submit( + JobTarget.colocated(TABLE_NAME, keyTuple), JobDescriptor.builder(NodeNameJob.class).build()); String expectedNode = "itcct_n_" + port; assertThat(tupleExecution.resultAsync(), willBe(expectedNode)); @@ -549,8 +549,8 @@ void testExecuteColocatedPojoRunsComputeJobOnKeyNode(int key, int port) { IgniteCompute igniteCompute = client().compute(); Mapper keyMapper = Mapper.of(TestPojo.class); - JobExecution pojoExecution = igniteCompute.submitColocated( - TABLE_NAME, keyPojo, keyMapper, JobDescriptor.builder(NodeNameJob.class).build()); + JobExecution pojoExecution = igniteCompute.submit( + JobTarget.colocated(TABLE_NAME, keyPojo, keyMapper), JobDescriptor.builder(NodeNameJob.class).build()); String expectedNode = "itcct_n_" + port; assertThat(pojoExecution.resultAsync(), willBe(expectedNode)); @@ -565,8 +565,8 @@ void testCancelColocatedTuple(int key, int port) { int sleepMs = 1_000_000; IgniteCompute igniteCompute = client().compute(); - JobExecution tupleExecution = igniteCompute.submitColocated( - TABLE_NAME, keyTuple, JobDescriptor.builder(SleepJob.class).build(), sleepMs); + JobExecution tupleExecution = igniteCompute.submit( + JobTarget.colocated(TABLE_NAME, keyTuple), JobDescriptor.builder(SleepJob.class).build(), sleepMs); await().until(tupleExecution::statusAsync, willBe(jobStatusWithState(EXECUTING))); @@ -583,8 +583,8 @@ void testCancelColocatedPojo(int key, int port) { IgniteCompute igniteCompute = client().compute(); Mapper keyMapper = Mapper.of(TestPojo.class); - JobExecution pojoExecution = igniteCompute.submitColocated( - TABLE_NAME, keyPojo, keyMapper, JobDescriptor.builder(SleepJob.class).build(), sleepMs); + JobExecution pojoExecution = igniteCompute.submit( + JobTarget.colocated(TABLE_NAME, keyPojo, keyMapper), JobDescriptor.builder(SleepJob.class).build(), sleepMs); await().until(pojoExecution::statusAsync, willBe(jobStatusWithState(EXECUTING))); @@ -615,9 +615,8 @@ void testExecuteOnUnknownUnitWithLatestVersionThrows() { void testExecuteColocatedOnUnknownUnitWithLatestVersionThrows() { CompletionException ex = assertThrows( CompletionException.class, - () -> client().compute().executeColocatedAsync( - TABLE_NAME, - Tuple.create().set(COLUMN_KEY, 1), + () -> client().compute().executeAsync( + JobTarget.colocated(TABLE_NAME, Tuple.create().set(COLUMN_KEY, 1)), JobDescriptor.builder(NodeNameJob.class) .units(new DeploymentUnit("u", "latest")) .build()).join()); @@ -673,9 +672,10 @@ void testExecuteColocatedEscapedTableName() { Mapper mapper = Mapper.of(TestPojo.class); TestPojo pojoKey = new TestPojo(1); Tuple tupleKey = Tuple.create().set("key", pojoKey.key); + JobDescriptor job = JobDescriptor.builder(NodeNameJob.class).build(); - var tupleRes = client().compute().executeColocated(tableName, tupleKey, JobDescriptor.builder(NodeNameJob.class).build()); - var pojoRes = client().compute().executeColocated(tableName, pojoKey, mapper, JobDescriptor.builder(NodeNameJob.class).build()); + var tupleRes = client().compute().execute(JobTarget.colocated(tableName, tupleKey), job); + var pojoRes = client().compute().execute(JobTarget.colocated(tableName, pojoKey, mapper), job); assertEquals(tupleRes, pojoRes); } From 7b8392c5ef7c3957fbf1f68ee1616aa53e68026c Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 09:28:24 +0300 Subject: [PATCH 23/52] ItThinClientComputeTest fixed --- .../app/client/ItThinClientPartitionAwarenessTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientPartitionAwarenessTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientPartitionAwarenessTest.java index ded59c2b37c..dc9b05cceac 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientPartitionAwarenessTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientPartitionAwarenessTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecutionContext; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.runner.app.client.proxy.IgniteClientProxy; import org.apache.ignite.table.RecordView; import org.apache.ignite.table.Tuple; @@ -87,8 +88,8 @@ void testGetRequestIsRoutedToPrimaryNode(boolean withTx) { for (int key = 0; key < 50; key++) { // Get actual primary node using compute. Tuple keyTuple = Tuple.create().set("key", key); - var primaryNodeName = proxyClient.compute().executeColocated( - TABLE_NAME, keyTuple, JobDescriptor.builder(NodeNameJob.class.getName()).build()); + var primaryNodeName = proxyClient.compute().execute( + JobTarget.colocated(TABLE_NAME, keyTuple), JobDescriptor.builder(NodeNameJob.class.getName()).build()); // Perform request and check routing with proxy. resetRequestCount(); From 37829a23e9db8b6d82d0e7b54c3b2d662fc42155 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 09:32:26 +0300 Subject: [PATCH 24/52] fixing client tests --- .../ignite/client/AbstractClientTest.java | 7 +++--- .../ignite/client/ClientComputeTest.java | 24 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java b/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java index baa404d409c..aa14b9dacd9 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java +++ b/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.client.fakes.FakeIgnite; import org.apache.ignite.client.fakes.FakeIgniteTables; import org.apache.ignite.client.fakes.FakeSchemaRegistry; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.client.ClientClusterNode; import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest; import org.apache.ignite.network.ClusterNode; @@ -186,9 +187,9 @@ public static IgniteClient getClient(TestServer... servers) { * @param names Names. * @return Nodes. */ - public static Set getClusterNodes(String... names) { - return Arrays.stream(names) + public static JobTarget getClusterNodes(String... names) { + return JobTarget.anyNode(Arrays.stream(names) .map(s -> new ClientClusterNode("id", s, new NetworkAddress("127.0.0.1", 8080))) - .collect(Collectors.toSet()); + .collect(Collectors.toSet())); } } diff --git a/modules/client/src/test/java/org/apache/ignite/client/ClientComputeTest.java b/modules/client/src/test/java/org/apache/ignite/client/ClientComputeTest.java index 3fa25176857..8f4f842eacd 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/ClientComputeTest.java +++ b/modules/client/src/test/java/org/apache/ignite/client/ClientComputeTest.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -51,6 +50,7 @@ import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.compute.version.Version; import org.apache.ignite.internal.client.table.ClientTable; @@ -58,7 +58,6 @@ import org.apache.ignite.internal.testframework.IgniteTestUtils; import org.apache.ignite.lang.IgniteException; import org.apache.ignite.lang.TableNotFoundException; -import org.apache.ignite.network.ClusterNode; import org.apache.ignite.table.Tuple; import org.apache.ignite.table.mapper.Mapper; import org.junit.jupiter.api.AfterEach; @@ -153,8 +152,8 @@ public void testExecuteColocated() throws Exception { try (var client = getClient(server2)) { JobDescriptor job = JobDescriptor.builder("job").build(); - String res1 = client.compute().executeColocated(TABLE_NAME, Tuple.create().set("key", "k"), job); - String res2 = client.compute().executeColocated(TABLE_NAME, 1L, Mapper.of(Long.class), job); + String res1 = client.compute().execute(JobTarget.colocated(TABLE_NAME, Tuple.create().set("key", "k")), job); + String res2 = client.compute().execute(JobTarget.colocated(TABLE_NAME, 1L, Mapper.of(Long.class)), job); assertEquals("s2", res1); assertEquals("s2", res2); @@ -168,8 +167,8 @@ public void testExecuteColocatedAsync() throws Exception { try (var client = getClient(server2)) { JobDescriptor job = JobDescriptor.builder("job").build(); - JobExecution execution1 = client.compute().submitColocated(TABLE_NAME, Tuple.create().set("key", "k"), job); - JobExecution execution2 = client.compute().submitColocated(TABLE_NAME, 1L, Mapper.of(Long.class), job); + JobExecution execution1 = client.compute().submit(JobTarget.colocated(TABLE_NAME, Tuple.create().set("key", "k")), job); + JobExecution execution2 = client.compute().submit(JobTarget.colocated(TABLE_NAME, 1L, Mapper.of(Long.class)), job); assertThat(execution1.resultAsync(), willBe("s2")); assertThat(execution2.resultAsync(), willBe("s2")); @@ -187,7 +186,8 @@ public void testExecuteColocatedThrowsTableNotFoundExceptionWhenTableDoesNotExis Tuple key = Tuple.create().set("key", "k"); var ex = assertThrows(CompletionException.class, - () -> client.compute().executeColocatedAsync("bad-tbl", key, JobDescriptor.builder("job").build()).join()); + () -> client.compute().executeAsync( + JobTarget.colocated("bad-tbl", key), JobDescriptor.builder("job").build()).join()); var tblNotFoundEx = (TableNotFoundException) ex.getCause(); assertThat(tblNotFoundEx.getMessage(), containsString("The table does not exist [name=\"PUBLIC\".\"bad-tbl\"]")); @@ -206,7 +206,7 @@ void testExecuteColocatedUpdatesTableCacheOnTableDrop(boolean forceLoadAssignmen try (var client = getClient(server3)) { Tuple key = Tuple.create().set("key", "k"); - String res1 = client.compute().executeColocated(tableName, key, JobDescriptor.builder("job").build()); + String res1 = client.compute().execute(JobTarget.colocated(tableName, key), JobDescriptor.builder("job").build()); // Drop table and create a new one with a different ID. ((FakeIgniteTables) ignite.tables()).dropTable(tableName); @@ -219,7 +219,9 @@ void testExecuteColocatedUpdatesTableCacheOnTableDrop(boolean forceLoadAssignmen IgniteTestUtils.setFieldValue(table, "partitionAssignment", null); } - String res2 = client.compute().executeColocated(tableName, 1L, Mapper.of(Long.class), JobDescriptor.builder("job").build()); + String res2 = client.compute().execute( + JobTarget.colocated(tableName, 1L, Mapper.of(Long.class)), + JobDescriptor.builder("job").build()); assertEquals("s3", res1); assertEquals("s3", res2); @@ -296,8 +298,8 @@ void testExceptionInJob() throws Exception { FakeCompute.future = CompletableFuture.failedFuture(new RuntimeException("job failed")); IgniteCompute igniteCompute = client.compute(); - Set nodes = getClusterNodes("s1"); - JobExecution execution = igniteCompute.submit(nodes, JobDescriptor.builder("job").build()); + var jobTarget = getClusterNodes("s1"); + JobExecution execution = igniteCompute.submit(jobTarget, JobDescriptor.builder("job").build()); assertThat(execution.resultAsync(), willThrowFast(IgniteException.class)); assertThat(execution.statusAsync(), willBe(jobStatusWithState(FAILED))); From 8eab1d7cd37cfc44220806ceaaaaf3e88c9a0805 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 09:38:35 +0300 Subject: [PATCH 25/52] fix FakeCompute --- .../ignite/client/fakes/FakeCompute.java | 65 +------------------ 1 file changed, 1 insertion(+), 64 deletions(-) diff --git a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java index b9e0830b5f2..1df5baf1773 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java +++ b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java @@ -59,7 +59,6 @@ import org.apache.ignite.internal.util.ExceptionUtils; import org.apache.ignite.network.ClusterNode; import org.apache.ignite.table.Tuple; -import org.apache.ignite.table.mapper.Mapper; import org.jetbrains.annotations.Nullable; /** @@ -134,19 +133,7 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob if (target instanceof NodesJobTarget) { return submit(((NodesJobTarget) target).nodes(), descriptor, args); } else if (target instanceof ColocatedExecutionTarget) { - ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; - var mapper = (Mapper) colocatedTarget.keyMapper(); - - if (mapper != null) { - return submitColocated( - colocatedTarget.tableName(), - colocatedTarget.key(), - mapper, - descriptor, - args); - } else { - return submitColocated(colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor, args); - } + return jobExecution(future != null ? future : completedFuture((R) nodeName)); } else { throw new IllegalArgumentException("Unsupported job target: " + target); } @@ -157,60 +144,10 @@ public R execute(JobTarget target, JobDescriptor descriptor, Object... args) return sync(this.executeAsync(target, descriptor, args)); } - @Override public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); } - @Override - public R execute(Set nodes, JobDescriptor descriptor, Object... args) { - return sync(this.executeAsync(nodes, descriptor, args)); - } - - @Override - public JobExecution submitColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return jobExecution(future != null ? future : completedFuture((R) nodeName)); - } - - @Override - public JobExecution submitColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return jobExecution(future != null ? future : completedFuture((R) nodeName)); - } - - /** {@inheritDoc} */ - @Override - public R executeColocated( - String tableName, - Tuple key, - JobDescriptor descriptor, - Object... args - ) { - return sync(this.executeColocatedAsync(tableName, key, descriptor, args)); - } - - /** {@inheritDoc} */ - @Override - public R executeColocated( - String tableName, - K key, - Mapper keyMapper, - JobDescriptor descriptor, - Object... args - ) { - return sync(executeColocatedAsync(tableName, key, keyMapper, descriptor, args)); - } - @Override public Map> submitBroadcast( Set nodes, From 84f17662df24055bbe43ba76acf0664ca30ad41b Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 11:09:10 +0300 Subject: [PATCH 26/52] Fix client tests --- .../org/apache/ignite/client/PartitionAwarenessTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/client/src/test/java/org/apache/ignite/client/PartitionAwarenessTest.java b/modules/client/src/test/java/org/apache/ignite/client/PartitionAwarenessTest.java index 984bd67db07..c556b9541c3 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/PartitionAwarenessTest.java +++ b/modules/client/src/test/java/org/apache/ignite/client/PartitionAwarenessTest.java @@ -40,6 +40,7 @@ import org.apache.ignite.client.handler.FakePlacementDriver; import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.client.ReliableChannel; import org.apache.ignite.internal.client.tx.ClientLazyTransaction; import org.apache.ignite.internal.hlc.HybridClockImpl; @@ -446,8 +447,8 @@ public void testExecuteColocatedTupleKeyRoutesRequestToPrimaryNode() { JobDescriptor job = JobDescriptor.builder("job").build(); - assertThat(compute().executeColocatedAsync(table.name(), t1, job), willBe(nodeKey1)); - assertThat(compute().executeColocatedAsync(table.name(), t2, job), willBe(nodeKey2)); + assertThat(compute().executeAsync(JobTarget.colocated(table.name(), t1), job), willBe(nodeKey1)); + assertThat(compute().executeAsync(JobTarget.colocated(table.name(), t2), job), willBe(nodeKey2)); } @Test @@ -456,8 +457,8 @@ public void testExecuteColocatedObjectKeyRoutesRequestToPrimaryNode() { Table table = defaultTable(); JobDescriptor job = JobDescriptor.builder("job").build(); - assertThat(compute().executeColocatedAsync(table.name(), 1L, mapper, job), willBe(nodeKey1)); - assertThat(compute().executeColocatedAsync(table.name(), 2L, mapper, job), willBe(nodeKey2)); + assertThat(compute().executeAsync(JobTarget.colocated(table.name(), 1L, mapper), job), willBe(nodeKey1)); + assertThat(compute().executeAsync(JobTarget.colocated(table.name(), 2L, mapper), job), willBe(nodeKey2)); } @ParameterizedTest From ce2bfddd18de278dcf9f85db068c36c49220dcb7 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 11:26:07 +0300 Subject: [PATCH 27/52] Fix embedded tests --- .../compute/ItComputeErrorsBaseTest.java | 4 ++-- .../compute/ItComputeTestEmbedded.java | 9 ++++---- .../compute/ItWorkerShutdownTest.java | 5 ++-- .../threading/ItComputeApiThreadingTest.java | 23 +++++++++++++++---- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java index 232db5333ab..82a6092e4a6 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeErrorsBaseTest.java @@ -90,7 +90,7 @@ void executeSucceedsWhenAtLeastOnNodeIsInTheCluster() { // And execute a job String workerNodeName = compute().execute( - nodes, + JobTarget.anyNode(nodes), JobDescriptor.builder(InteractiveJobs.globalJob().name()).build(), RETURN_WORKER_NAME.name()); @@ -101,7 +101,7 @@ void executeSucceedsWhenAtLeastOnNodeIsInTheCluster() { @Test void executeFailsWhenNoNodesAreInTheCluster() { // When set of nodes contain only non-existing nodes - Set nodes = Set.of(nonExistingNode); + JobTarget nodes = JobTarget.node(nonExistingNode); // Then job fails. assertThrows( diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java index 1b0e9a07fea..e61eca183a1 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java @@ -40,7 +40,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -230,7 +229,7 @@ void shouldNotConvertIgniteException() { IgniteException exception = new IgniteException(INTERNAL_ERR, "Test exception"); IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(CustomFailingJob.class).units(units()).build(), exception)); @@ -244,7 +243,7 @@ void shouldNotConvertIgniteCheckedException() { IgniteCheckedException exception = new IgniteCheckedException(INTERNAL_ERR, "Test exception"); IgniteCheckedException ex = assertThrows(IgniteCheckedException.class, () -> entryNode.compute().execute( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(CustomFailingJob.class).units(units()).build(), exception)); @@ -266,7 +265,7 @@ void shouldConvertToComputeException(Throwable throwable) { IgniteImpl entryNode = node(0); IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute( - Set.of(entryNode.node()), + JobTarget.node(entryNode.node()), JobDescriptor.builder(CustomFailingJob.class).units(units()).build(), throwable)); @@ -284,7 +283,7 @@ void executesSyncKvGetPutFromJob(int targetNodeIndex) { IgniteImpl targetNode = node(targetNodeIndex); assertDoesNotThrow(() -> entryNode.compute().execute( - Set.of(targetNode.node()), + JobTarget.node(targetNode.node()), JobDescriptor.builder(PerformSyncKvGetPutJob.class).build())); } diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java index 758782a1b0e..a83b9a4fe41 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItWorkerShutdownTest.java @@ -291,9 +291,8 @@ void colocatedExecutionWorkerShutdown() throws Exception { // When start colocated job on node that is not primary replica. IgniteImpl entryNode = anyNodeExcept(primaryReplica); TestingJobExecution execution = new TestingJobExecution<>( - compute(entryNode).submitColocated( - TABLE_NAME, - Tuple.create(1).set("K", 1), + compute(entryNode).submit( + JobTarget.colocated(TABLE_NAME, Tuple.create(1).set("K", 1)), JobDescriptor.builder(InteractiveJobs.globalJob().name()).build())); // Then the job is alive. diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java index a377163fb42..739fd8aa3c6 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/threading/ItComputeApiThreadingTest.java @@ -150,10 +150,17 @@ public CompletableFuture executeAsync(JobExecutionContext context, Objec private enum ComputeAsyncOperation { EXECUTE_ASYNC(compute -> compute.executeAsync(JobTarget.anyNode(justNonEntryNode()), JobDescriptor.builder(NoOpJob.class).build())), + EXECUTE_COLOCATED_BY_TUPLE_ASYNC(compute -> - compute.executeColocatedAsync(TABLE_NAME, KEY_TUPLE, JobDescriptor.builder(NoOpJob.class).build())), + compute.executeAsync( + JobTarget.colocated(TABLE_NAME, KEY_TUPLE), + JobDescriptor.builder(NoOpJob.class).build())), + EXECUTE_COLOCATED_BY_KEY_ASYNC(compute -> - compute.executeColocatedAsync(TABLE_NAME, KEY, Mapper.of(Integer.class), JobDescriptor.builder(NoOpJob.class).build())), + compute.executeAsync( + JobTarget.colocated(TABLE_NAME, KEY, Mapper.of(Integer.class)), + JobDescriptor.builder(NoOpJob.class).build())), + EXECUTE_BROADCAST_ASYNC(compute -> compute.executeBroadcastAsync(justNonEntryNode(), JobDescriptor.builder(NoOpJob.class).build())); private final Function> action; @@ -169,10 +176,16 @@ CompletableFuture executeOn(IgniteCompute compute) { private enum ComputeSubmitOperation { SUBMIT(compute -> compute.submit(JobTarget.anyNode(justNonEntryNode()), JobDescriptor.builder(NoOpJob.class).build())), - SUBMIT_COLOCATED_BY_TUPLE(compute -> compute.submitColocated(TABLE_NAME, KEY_TUPLE, JobDescriptor.builder(NoOpJob.class).build())), - SUBMIT_COLOCATED_BY_KEY(compute -> compute.submitColocated( - TABLE_NAME, KEY, Mapper.of(Integer.class), JobDescriptor.builder(NoOpJob.class).build()) + + SUBMIT_COLOCATED_BY_TUPLE(compute -> compute.submit( + JobTarget.colocated(TABLE_NAME, KEY_TUPLE), + JobDescriptor.builder(NoOpJob.class).build())), + + SUBMIT_COLOCATED_BY_KEY(compute -> compute.submit( + JobTarget.colocated(TABLE_NAME, KEY, Mapper.of(Integer.class)), + JobDescriptor.builder(NoOpJob.class).build()) ), + SUBMIT_BROADCAST(compute -> compute .submitBroadcast(justNonEntryNode(), JobDescriptor.builder(NoOpJob.class).build()) .values().iterator().next() From 18a01f0dc713bd8eb0efa2cbec8f06594208fbb3 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 11:27:26 +0300 Subject: [PATCH 28/52] wip --- .../org/apache/ignite/compute/ColocatedExecutionTarget.java | 2 +- .../main/java/org/apache/ignite/compute/NodesJobTarget.java | 2 +- .../ignite/internal/rest/compute/ItComputeControllerTest.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java index 8f8d0842ed5..ea046f98f08 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java @@ -32,7 +32,7 @@ public class ColocatedExecutionTarget implements JobTarget { private final @Nullable Mapper keyMapper; - public ColocatedExecutionTarget(String tableName, Object key, @Nullable Mapper keyMapper) { + ColocatedExecutionTarget(String tableName, Object key, @Nullable Mapper keyMapper) { Objects.requireNonNull(tableName); Objects.requireNonNull(key); diff --git a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java index 5b289b75254..ab12ff76902 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java @@ -27,7 +27,7 @@ public class NodesJobTarget implements JobTarget { private final Set nodes; - public NodesJobTarget(Set nodes) { + NodesJobTarget(Set nodes) { Objects.requireNonNull(nodes); if (nodes.isEmpty()) { diff --git a/modules/rest/src/integrationTest/java/org/apache/ignite/internal/rest/compute/ItComputeControllerTest.java b/modules/rest/src/integrationTest/java/org/apache/ignite/internal/rest/compute/ItComputeControllerTest.java index 767fbc4a15a..17851fb0c57 100644 --- a/modules/rest/src/integrationTest/java/org/apache/ignite/internal/rest/compute/ItComputeControllerTest.java +++ b/modules/rest/src/integrationTest/java/org/apache/ignite/internal/rest/compute/ItComputeControllerTest.java @@ -46,6 +46,7 @@ import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionContext; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.rest.api.Problem; @@ -365,7 +366,7 @@ void shouldReturnFalseIfUpdatePriorityOfCompletedJob() { } private static JobExecution runBlockingJob(IgniteImpl entryNode, Set nodes) { - return entryNode.compute().submit(nodes, JobDescriptor.builder(BlockingJob.class).build()); + return entryNode.compute().submit(JobTarget.anyNode(nodes), JobDescriptor.builder(BlockingJob.class).build()); } private static void unblockJob() { From 53527ffd289a873910802eb514838120011e4215 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 11:30:39 +0300 Subject: [PATCH 29/52] fixing checkstyle --- .../ignite/client/AbstractClientTest.java | 2 -- .../ignite/client/fakes/FakeCompute.java | 7 ++----- .../app/client/ItThinClientComputeTest.java | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java b/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java index aa14b9dacd9..3c938a6b871 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java +++ b/modules/client/src/test/java/org/apache/ignite/client/AbstractClientTest.java @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import java.util.Arrays; -import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import org.apache.ignite.Ignite; @@ -32,7 +31,6 @@ import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.client.ClientClusterNode; import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest; -import org.apache.ignite.network.ClusterNode; import org.apache.ignite.network.NetworkAddress; import org.apache.ignite.table.Tuple; import org.junit.jupiter.api.AfterAll; diff --git a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java index 1df5baf1773..3911cb0229a 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java +++ b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java @@ -131,7 +131,8 @@ public CompletableFuture> submitColocatedInternal(TableViewI @Override public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { if (target instanceof NodesJobTarget) { - return submit(((NodesJobTarget) target).nodes(), descriptor, args); + Set nodes = ((NodesJobTarget) target).nodes(); + return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); } else if (target instanceof ColocatedExecutionTarget) { return jobExecution(future != null ? future : completedFuture((R) nodeName)); } else { @@ -144,10 +145,6 @@ public R execute(JobTarget target, JobDescriptor descriptor, Object... args) return sync(this.executeAsync(target, descriptor, args)); } - public JobExecution submit(Set nodes, JobDescriptor descriptor, Object... args) { - return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); - } - @Override public Map> submitBroadcast( Set nodes, diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java index 993d131f4ac..127a95e00a5 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/client/ItThinClientComputeTest.java @@ -145,7 +145,9 @@ void testExecuteOnSpecificNodeAsync() { @Test void testCancellingCompletedJob() { - JobExecution execution = client().compute().submit(JobTarget.node(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); + JobExecution execution = client().compute().submit( + JobTarget.node(node(0)), + JobDescriptor.builder(NodeNameJob.class).build()); assertThat(execution.resultAsync(), willBe("itcct_n_3344")); @@ -156,7 +158,9 @@ void testCancellingCompletedJob() { @Test void testChangingPriorityCompletedJob() { - JobExecution execution = client().compute().submit(JobTarget.node(node(0)), JobDescriptor.builder(NodeNameJob.class).build()); + JobExecution execution = client().compute().submit( + JobTarget.node(node(0)), + JobDescriptor.builder(NodeNameJob.class).build()); assertThat(execution.resultAsync(), willBe("itcct_n_3344")); @@ -455,8 +459,9 @@ void testExceptionInColocatedPojoJobPropagatesToClientWithClassAndMessageSync() var key = new TestPojo(1); Mapper mapper = Mapper.of(TestPojo.class); - IgniteException cause = getExceptionInJobExecutionSync( - () -> client().compute().execute(JobTarget.colocated(TABLE_NAME, key, mapper), JobDescriptor.builder(ExceptionJob.class).build()) + IgniteException cause = getExceptionInJobExecutionSync(() -> client().compute().execute( + JobTarget.colocated(TABLE_NAME, key, mapper), + JobDescriptor.builder(ExceptionJob.class).build()) ); assertComputeExceptionWithClassAndMessage(cause); @@ -683,7 +688,11 @@ void testExecuteColocatedEscapedTableName() { @ParameterizedTest @CsvSource({"1E3,-3", "1.12E5,-5", "1.12E5,0", "1.123456789,10", "1.123456789,5"}) void testBigDecimalPropagation(String number, int scale) { - BigDecimal res = client().compute().execute(JobTarget.node(node(0)), JobDescriptor.builder(DecimalJob.class).build(), number, scale); + BigDecimal res = client().compute().execute( + JobTarget.node(node(0)), + JobDescriptor.builder(DecimalJob.class).build(), + number, + scale); var expected = new BigDecimal(number).setScale(scale, RoundingMode.HALF_UP); assertEquals(expected, res); From 9780a407e473ad95b51aecd4e23e4f62591a923a Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 11:31:47 +0300 Subject: [PATCH 30/52] Fix IgniteComputeImplTest --- .../compute/IgniteComputeImplTest.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/modules/compute/src/test/java/org/apache/ignite/internal/compute/IgniteComputeImplTest.java b/modules/compute/src/test/java/org/apache/ignite/internal/compute/IgniteComputeImplTest.java index 4cdf6f49c50..d2b03c2b43b 100644 --- a/modules/compute/src/test/java/org/apache/ignite/internal/compute/IgniteComputeImplTest.java +++ b/modules/compute/src/test/java/org/apache/ignite/internal/compute/IgniteComputeImplTest.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.compute; -import static java.util.Collections.singleton; import static java.util.concurrent.CompletableFuture.completedFuture; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willBe; import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture; @@ -44,6 +43,7 @@ import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionOptions; import org.apache.ignite.compute.JobStatus; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologyService; import org.apache.ignite.internal.hlc.HybridClock; import org.apache.ignite.internal.network.ClusterNodeImpl; @@ -112,7 +112,7 @@ void whenNodeIsLocalThenExecutesLocally() { assertThat( compute.executeAsync( - singleton(localNode), + JobTarget.node(localNode), JobDescriptor.builder(JOB_CLASS_NAME).units(testDeploymentUnits).build(), "a", 42), willBe("jobResponse") @@ -127,7 +127,7 @@ void whenNodeIsRemoteThenExecutesRemotely() { assertThat( compute.executeAsync( - singleton(remoteNode), + JobTarget.node(remoteNode), JobDescriptor.builder(JOB_CLASS_NAME).units(testDeploymentUnits).build(), "a", 42), willBe("remoteResponse") @@ -144,7 +144,7 @@ void whenNodeIsLocalThenExecutesLocallyWithOptions() { JobExecutionOptions options = JobExecutionOptions.builder().priority(1).maxRetries(2).build(); assertThat( compute.executeAsync( - singleton(localNode), + JobTarget.node(localNode), JobDescriptor.builder(JOB_CLASS_NAME).units(testDeploymentUnits).options(options).build(), "a", 42), willBe("jobResponse") @@ -162,7 +162,7 @@ void whenNodeIsRemoteThenExecutesRemotelyWithOptions() { assertThat( compute.executeAsync( - singleton(remoteNode), + JobTarget.node(remoteNode), JobDescriptor.builder(JOB_CLASS_NAME).units(testDeploymentUnits).options(options).build(), "a", 42), willBe("remoteResponse") @@ -177,9 +177,8 @@ void executesColocatedOnLeaderNodeOfPartitionCorrespondingToTupleKey() { respondWhenAskForPrimaryReplica(); assertThat( - compute.executeColocatedAsync( - "test", - Tuple.create(Map.of("k", 1)), + compute.executeAsync( + JobTarget.colocated("test", Tuple.create(Map.of("k", 1))), JobDescriptor.builder(JOB_CLASS_NAME).units(testDeploymentUnits).build(), "a", 42), willBe("remoteResponse") @@ -192,10 +191,8 @@ void executesColocatedOnLeaderNodeOfPartitionCorrespondingToMappedKey() { respondWhenAskForPrimaryReplica(); assertThat( - compute.executeColocatedAsync( - "test", - 1, - Mapper.of(Integer.class), + compute.executeAsync( + JobTarget.colocated("test", 1, Mapper.of(Integer.class)), JobDescriptor.builder(JOB_CLASS_NAME).units(testDeploymentUnits).build(), "a", 42 ), From b46471d94ad446a97d59fff83717719d00570d33 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 11:36:16 +0300 Subject: [PATCH 31/52] Fix merge --- .../apache/ignite/internal/compute/ItComputeTestEmbedded.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java index e299342fd00..e1d56aab298 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java @@ -55,7 +55,6 @@ import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionContext; import org.apache.ignite.compute.JobExecutionOptions; -import org.apache.ignite.compute.JobState; import org.apache.ignite.compute.JobTarget; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.lang.IgniteInternalCheckedException; @@ -99,7 +98,7 @@ void cancelsJobLocally() { @Test void cancelsQueuedJobLocally() { IgniteImpl entryNode = node(0); - Set nodes = Set.of(entryNode.node()); + var nodes = JobTarget.node(entryNode.node()); CountDownLatch countDownLatch = new CountDownLatch(1); JobDescriptor job = JobDescriptor.builder(WaitLatchJob.class).units(units()).build(); From ae790b4745c759da9c8f405e9dda696496dc6d0a Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 12:16:42 +0300 Subject: [PATCH 32/52] Fix javadoc --- .../main/java/org/apache/ignite/compute/task/MapReduceJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/task/MapReduceJob.java b/modules/api/src/main/java/org/apache/ignite/compute/task/MapReduceJob.java index c3df006c382..8b9e7a90b5f 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/task/MapReduceJob.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/task/MapReduceJob.java @@ -22,11 +22,12 @@ import java.util.HashSet; import java.util.Set; import org.apache.ignite.compute.JobDescriptor; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.network.ClusterNode; /** * A description of the job to be submitted as a result of the split step of the {@link MapReduceTask}. Reflects the parameters of the - * {@link org.apache.ignite.compute.IgniteCompute#submit(Set, JobDescriptor, Object...)} method. + * {@link org.apache.ignite.compute.IgniteCompute#submit(JobTarget, JobDescriptor, Object...)} method. */ public class MapReduceJob { private final Set nodes; From 1c178760e85ced072a63a5afaabb1a533f48c1d8 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 19 Jun 2024 12:17:27 +0300 Subject: [PATCH 33/52] Fix javadoc --- .../java/org/apache/ignite/client/ClientOperationType.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/client/ClientOperationType.java b/modules/client/src/main/java/org/apache/ignite/client/ClientOperationType.java index 459d88303db..8c6106097ee 100644 --- a/modules/client/src/main/java/org/apache/ignite/client/ClientOperationType.java +++ b/modules/client/src/main/java/org/apache/ignite/client/ClientOperationType.java @@ -19,8 +19,8 @@ import java.util.Collection; import java.util.List; -import java.util.Set; import org.apache.ignite.compute.JobDescriptor; +import org.apache.ignite.compute.JobTarget; import org.apache.ignite.sql.BatchedArguments; import org.apache.ignite.sql.IgniteSql; import org.apache.ignite.sql.async.AsyncResultSet; @@ -130,7 +130,7 @@ public enum ClientOperationType { TUPLE_CONTAINS_KEY, /** - * Compute Execute ({@link org.apache.ignite.compute.IgniteCompute#submit(Set, JobDescriptor, Object...)}). + * Compute Execute ({@link org.apache.ignite.compute.IgniteCompute#submit(JobTarget, JobDescriptor, Object...)}). */ COMPUTE_EXECUTE, From 03c150aac3181278cf31d3139fc3d19530d94b56 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 08:23:18 +0300 Subject: [PATCH 34/52] Update modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java Co-authored-by: Vadim Pakhnushev <8614891+valepakh@users.noreply.github.com> --- .../apache/ignite/internal/client/compute/ClientCompute.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index 6c0dded325f..d6898a00786 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -97,8 +97,8 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob if (target instanceof NodesJobTarget) { return new ClientJobExecution<>( ch, - executeOnNodesAsync(( - (NodesJobTarget) target).nodes(), + executeOnNodesAsync( + ((NodesJobTarget) target).nodes(), descriptor.units(), descriptor.jobClassName(), descriptor.options(), From 315b57daa7f2337282fbffeadd11d3a9bc93bdab Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 08:25:47 +0300 Subject: [PATCH 35/52] ClientCompute cleanup --- .../internal/client/compute/ClientCompute.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index d6898a00786..7b62b7b7ef6 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -95,10 +95,10 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob Objects.requireNonNull(descriptor); if (target instanceof NodesJobTarget) { - return new ClientJobExecution<>( - ch, - executeOnNodesAsync( - ((NodesJobTarget) target).nodes(), + NodesJobTarget nodesJobTarget = (NodesJobTarget) target; + + return new ClientJobExecution<>(ch, executeOnNodesAsync( + nodesJobTarget.nodes(), descriptor.units(), descriptor.jobClassName(), descriptor.options(), @@ -119,9 +119,7 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob descriptor.options(), args)); } else { - return new ClientJobExecution<>( - ch, - doExecuteColocatedAsync( + return new ClientJobExecution<>(ch, doExecuteColocatedAsync( colocatedTarget.tableName(), (Tuple) colocatedTarget.key(), descriptor.units(), @@ -412,6 +410,7 @@ private static void packTask(ClientMessagePacker w, * @return Result of the job submission. */ private static SubmitResult unpackSubmitResult(PayloadInputChannel ch) { + //noinspection DataFlowIssue (reviewed) return new SubmitResult(ch.in().unpackUuid(), ch.notificationFuture()); } @@ -432,6 +431,7 @@ private static SubmitTaskResult unpackSubmitTaskResult(PayloadInputChannel ch) { jobIds.add(ch.in().unpackUuid()); } + //noinspection DataFlowIssue (reviewed) return new SubmitTaskResult(jobId, jobIds, ch.notificationFuture()); } From 896772131b9aa075522628b93027dcea85702f6b Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 08:26:16 +0300 Subject: [PATCH 36/52] Update modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java Co-authored-by: Vadim Pakhnushev <8614891+valepakh@users.noreply.github.com> --- .../test/java/org/apache/ignite/client/fakes/FakeCompute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java index 3911cb0229a..78a478ca685 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java +++ b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java @@ -142,7 +142,7 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob @Override public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { - return sync(this.executeAsync(target, descriptor, args)); + return sync(executeAsync(target, descriptor, args)); } @Override From 33965891a99b3746b714c36f74cd52a40868b423 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 08:26:27 +0300 Subject: [PATCH 37/52] Update modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java Co-authored-by: Vadim Pakhnushev <8614891+valepakh@users.noreply.github.com> --- .../org/apache/ignite/internal/compute/ItComputeBaseTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java index 0f39ff53763..59866d2d62b 100644 --- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java +++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java @@ -121,7 +121,7 @@ void executesWrongJobClassOnRemoteNodesAsync(String jobClassName, int errorCode, Ignite entryNode = node(0); ExecutionException ex = assertThrows(ExecutionException.class, () -> entryNode.compute().executeAsync( - JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(jobClassName).units(units()).build()) + JobTarget.anyNode(node(1).node(), node(2).node()), JobDescriptor.builder(jobClassName).units(units()).build()) .get(1, TimeUnit.SECONDS)); assertTraceableException(ex, ComputeException.class, errorCode, msg); From d8e2da32f5486ee099032a5bc6a7a4d464413a1c Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 08:26:36 +0300 Subject: [PATCH 38/52] Update modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java Co-authored-by: Vadim Pakhnushev <8614891+valepakh@users.noreply.github.com> --- .../org/apache/ignite/internal/compute/IgniteComputeImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index 24efe43fbae..f24ba87c92e 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -146,7 +146,7 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob @Override public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { - return sync(this.submit(target, descriptor, args).resultAsync()); + return sync(executeAsync(target, descriptor, args)); } @Override From a678027084b13224d298963a61a13c3c65cc9008 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:39:46 +0300 Subject: [PATCH 39/52] Rename job targets --- .../{NodesJobTarget.java => AnyNodeJobTarget.java} | 4 ++-- ...xecutionTarget.java => ColocatedJobTarget.java} | 4 ++-- .../java/org/apache/ignite/compute/JobTarget.java | 12 ++++++------ .../internal/client/compute/ClientCompute.java | 14 +++++++------- .../apache/ignite/client/fakes/FakeCompute.java | 10 +++++----- .../ignite/internal/compute/IgniteComputeImpl.java | 12 ++++++------ 6 files changed, 28 insertions(+), 28 deletions(-) rename modules/api/src/main/java/org/apache/ignite/compute/{NodesJobTarget.java => AnyNodeJobTarget.java} (92%) rename modules/api/src/main/java/org/apache/ignite/compute/{ColocatedExecutionTarget.java => ColocatedJobTarget.java} (91%) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/AnyNodeJobTarget.java similarity index 92% rename from modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java rename to modules/api/src/main/java/org/apache/ignite/compute/AnyNodeJobTarget.java index ab12ff76902..beb834b78d5 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/NodesJobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/AnyNodeJobTarget.java @@ -24,10 +24,10 @@ /** * Nodes-based job execution target. */ -public class NodesJobTarget implements JobTarget { +public class AnyNodeJobTarget implements JobTarget { private final Set nodes; - NodesJobTarget(Set nodes) { + AnyNodeJobTarget(Set nodes) { Objects.requireNonNull(nodes); if (nodes.isEmpty()) { diff --git a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedJobTarget.java similarity index 91% rename from modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java rename to modules/api/src/main/java/org/apache/ignite/compute/ColocatedJobTarget.java index ea046f98f08..861e5b7c907 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedExecutionTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedJobTarget.java @@ -25,14 +25,14 @@ /** * Colocated job execution target. */ -public class ColocatedExecutionTarget implements JobTarget { +public class ColocatedJobTarget implements JobTarget { private final String tableName; private final Object key; private final @Nullable Mapper keyMapper; - ColocatedExecutionTarget(String tableName, Object key, @Nullable Mapper keyMapper) { + ColocatedJobTarget(String tableName, Object key, @Nullable Mapper keyMapper) { Objects.requireNonNull(tableName); Objects.requireNonNull(key); diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index e0654f12429..8685b23d8cb 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -35,7 +35,7 @@ public interface JobTarget { * @return Job target. */ static JobTarget node(ClusterNode node) { - return new NodesJobTarget(Set.of(node)); + return new AnyNodeJobTarget(Set.of(node)); } /** @@ -45,7 +45,7 @@ static JobTarget node(ClusterNode node) { * @return Job target. */ static JobTarget anyNode(ClusterNode... nodes) { - return new NodesJobTarget(Set.of(nodes)); + return new AnyNodeJobTarget(Set.of(nodes)); } /** @@ -55,7 +55,7 @@ static JobTarget anyNode(ClusterNode... nodes) { * @return Job target. */ static JobTarget anyNode(Collection nodes) { - return new NodesJobTarget(new HashSet<>(nodes)); + return new AnyNodeJobTarget(new HashSet<>(nodes)); } /** @@ -65,7 +65,7 @@ static JobTarget anyNode(Collection nodes) { * @return Job target. */ static JobTarget anyNode(Set nodes) { - return new NodesJobTarget(nodes); + return new AnyNodeJobTarget(nodes); } /** @@ -76,7 +76,7 @@ static JobTarget anyNode(Set nodes) { * @return Job target. */ static JobTarget colocated(String tableName, Tuple key) { - return new ColocatedExecutionTarget(tableName, key, null); + return new ColocatedJobTarget(tableName, key, null); } /** @@ -87,6 +87,6 @@ static JobTarget colocated(String tableName, Tuple key) { * @return Job target. */ static JobTarget colocated(String tableName, K key, Mapper keyMapper) { - return new ColocatedExecutionTarget(tableName, key, keyMapper); + return new ColocatedJobTarget(tableName, key, keyMapper); } } diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index 7b62b7b7ef6..7afca6f91ac 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -35,14 +35,14 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; -import org.apache.ignite.compute.ColocatedExecutionTarget; +import org.apache.ignite.compute.ColocatedJobTarget; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.IgniteCompute; import org.apache.ignite.compute.JobDescriptor; import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionOptions; import org.apache.ignite.compute.JobTarget; -import org.apache.ignite.compute.NodesJobTarget; +import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.client.ClientUtils; import org.apache.ignite.internal.client.PayloadInputChannel; @@ -94,19 +94,19 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob Objects.requireNonNull(target); Objects.requireNonNull(descriptor); - if (target instanceof NodesJobTarget) { - NodesJobTarget nodesJobTarget = (NodesJobTarget) target; + if (target instanceof AnyNodeJobTarget) { + AnyNodeJobTarget anyNodeJobTarget = (AnyNodeJobTarget) target; return new ClientJobExecution<>(ch, executeOnNodesAsync( - nodesJobTarget.nodes(), + anyNodeJobTarget.nodes(), descriptor.units(), descriptor.jobClassName(), descriptor.options(), args)); } - if (target instanceof ColocatedExecutionTarget) { - ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; + if (target instanceof ColocatedJobTarget) { + ColocatedJobTarget colocatedTarget = (ColocatedJobTarget) target; var mapper = (Mapper) colocatedTarget.keyMapper(); if (mapper != null) { diff --git a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java index 78a478ca685..c874d1acea7 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java +++ b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java @@ -39,7 +39,7 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; import org.apache.ignite.Ignite; -import org.apache.ignite.compute.ColocatedExecutionTarget; +import org.apache.ignite.compute.ColocatedJobTarget; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.IgniteCompute; @@ -49,7 +49,7 @@ import org.apache.ignite.compute.JobState; import org.apache.ignite.compute.JobStatus; import org.apache.ignite.compute.JobTarget; -import org.apache.ignite.compute.NodesJobTarget; +import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.compute.ComputeUtils; import org.apache.ignite.internal.compute.IgniteComputeInternal; @@ -130,10 +130,10 @@ public CompletableFuture> submitColocatedInternal(TableViewI @Override public JobExecution submit(JobTarget target, JobDescriptor descriptor, Object... args) { - if (target instanceof NodesJobTarget) { - Set nodes = ((NodesJobTarget) target).nodes(); + if (target instanceof AnyNodeJobTarget) { + Set nodes = ((AnyNodeJobTarget) target).nodes(); return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); - } else if (target instanceof ColocatedExecutionTarget) { + } else if (target instanceof ColocatedJobTarget) { return jobExecution(future != null ? future : completedFuture((R) nodeName)); } else { throw new IllegalArgumentException("Unsupported job target: " + target); diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index f24ba87c92e..c22b5797892 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -39,7 +39,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.apache.ignite.compute.ColocatedExecutionTarget; +import org.apache.ignite.compute.ColocatedJobTarget; import org.apache.ignite.compute.ComputeException; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.IgniteCompute; @@ -49,7 +49,7 @@ import org.apache.ignite.compute.JobStatus; import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.NodeNotFoundException; -import org.apache.ignite.compute.NodesJobTarget; +import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.task.MapReduceJob; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.hlc.HybridClock; @@ -102,14 +102,14 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob Objects.requireNonNull(target); Objects.requireNonNull(descriptor); - if (target instanceof NodesJobTarget) { - Set nodes = ((NodesJobTarget) target).nodes(); + if (target instanceof AnyNodeJobTarget) { + Set nodes = ((AnyNodeJobTarget) target).nodes(); return executeAsyncWithFailover(nodes, descriptor.units(), descriptor.jobClassName(), descriptor.options(), args); } - if (target instanceof ColocatedExecutionTarget) { - ColocatedExecutionTarget colocatedTarget = (ColocatedExecutionTarget) target; + if (target instanceof ColocatedJobTarget) { + ColocatedJobTarget colocatedTarget = (ColocatedJobTarget) target; var mapper = (Mapper) colocatedTarget.keyMapper(); String tableName = colocatedTarget.tableName(); Object key = colocatedTarget.key(); From e907ed29cee52113cb0c7d88b1571901b8456a85 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:40:17 +0300 Subject: [PATCH 40/52] wip naming --- .../ignite/internal/client/compute/ClientCompute.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index 7afca6f91ac..b981cc7df14 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -97,7 +97,7 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob if (target instanceof AnyNodeJobTarget) { AnyNodeJobTarget anyNodeJobTarget = (AnyNodeJobTarget) target; - return new ClientJobExecution<>(ch, executeOnNodesAsync( + return new ClientJobExecution<>(ch, executeOnRandomNodeAsync( anyNodeJobTarget.nodes(), descriptor.units(), descriptor.jobClassName(), @@ -189,7 +189,7 @@ public Map> submitBroadcast( Map> map = new HashMap<>(nodes.size()); for (ClusterNode node : nodes) { - JobExecution execution = new ClientJobExecution<>(ch, executeOnNodesAsync( + JobExecution execution = new ClientJobExecution<>(ch, executeOnRandomNodeAsync( Set.of(node), descriptor.units(), descriptor.jobClassName(), descriptor.options(), args )); if (map.put(node, execution) != null) { @@ -227,7 +227,7 @@ private CompletableFuture doExecuteMapReduceAsync( ); } - private CompletableFuture executeOnNodesAsync( + private CompletableFuture executeOnRandomNodeAsync( Set nodes, List units, String jobClassName, From ca8925184f8b6651890f2f8650edf4f8b92877af Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:41:22 +0300 Subject: [PATCH 41/52] wip naming --- .../ignite/internal/client/compute/ClientCompute.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index b981cc7df14..bcaeae90e9f 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -97,7 +97,7 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob if (target instanceof AnyNodeJobTarget) { AnyNodeJobTarget anyNodeJobTarget = (AnyNodeJobTarget) target; - return new ClientJobExecution<>(ch, executeOnRandomNodeAsync( + return new ClientJobExecution<>(ch, executeOnAnyNodeAsync( anyNodeJobTarget.nodes(), descriptor.units(), descriptor.jobClassName(), @@ -189,7 +189,7 @@ public Map> submitBroadcast( Map> map = new HashMap<>(nodes.size()); for (ClusterNode node : nodes) { - JobExecution execution = new ClientJobExecution<>(ch, executeOnRandomNodeAsync( + JobExecution execution = new ClientJobExecution<>(ch, executeOnAnyNodeAsync( Set.of(node), descriptor.units(), descriptor.jobClassName(), descriptor.options(), args )); if (map.put(node, execution) != null) { @@ -227,7 +227,7 @@ private CompletableFuture doExecuteMapReduceAsync( ); } - private CompletableFuture executeOnRandomNodeAsync( + private CompletableFuture executeOnAnyNodeAsync( Set nodes, List units, String jobClassName, From f69abad12bbc2da0e189d92bd4588b98ee16a106 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:42:08 +0300 Subject: [PATCH 42/52] Update modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java Co-authored-by: korlov42 --- .../api/src/main/java/org/apache/ignite/compute/JobTarget.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index 8685b23d8cb..40b0fe15947 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -26,6 +26,8 @@ /** * Job execution target. + * + *

Determines the rules for selecting nodes to execute a job. */ public interface JobTarget { /** From cb3d9747a48aae0371a96ea58da2574bbba4e985 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:42:30 +0300 Subject: [PATCH 43/52] Update modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java Co-authored-by: korlov42 --- .../api/src/main/java/org/apache/ignite/compute/JobTarget.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index 40b0fe15947..3676dac1c90 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -33,6 +33,8 @@ public interface JobTarget { /** * Creates a job target for a specific node. * + *

This target determines that a job should be executed on a given node. + * * @param node Node. * @return Job target. */ From 15b3f92b5dd2fce36dc15e1dc20e032c8b3a9ede Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:43:03 +0300 Subject: [PATCH 44/52] Update modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java Co-authored-by: korlov42 --- .../api/src/main/java/org/apache/ignite/compute/JobTarget.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index 3676dac1c90..4703d02b355 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -45,6 +45,9 @@ static JobTarget node(ClusterNode node) { /** * Creates a job target for any node from the provided collection. * + *

This target determines that a job can be executed on any node in a given collection, but only one of them. + * Which node is chosen is implementation defined. + * * @param nodes Collection of nodes. * @return Job target. */ From 305ee22943a8d76e5b54cacc4f3ed6a500cb0412 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:43:13 +0300 Subject: [PATCH 45/52] Update modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java Co-authored-by: korlov42 --- .../api/src/main/java/org/apache/ignite/compute/JobTarget.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index 4703d02b355..ba5ded3ab4c 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -58,6 +58,9 @@ static JobTarget anyNode(ClusterNode... nodes) { /** * Creates a job target for any node from the provided collection. * + *

This target determines that a job can be executed on any node in a given collection, but only one of them. + * Which node is chosen is implementation defined. + * * @param nodes Collection of nodes. * @return Job target. */ From cd5e750f45d5b20a9971b7bb606b04a4150864df Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:43:20 +0300 Subject: [PATCH 46/52] Update modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java Co-authored-by: korlov42 --- .../api/src/main/java/org/apache/ignite/compute/JobTarget.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index ba5ded3ab4c..85745ca383f 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -71,6 +71,9 @@ static JobTarget anyNode(Collection nodes) { /** * Creates a job target for any node from the provided collection. * + *

This target determines that a job can be executed on any node in a given collection, but only one of them. + * Which node is chosen is implementation defined. + * * @param nodes Collection of nodes. * @return Job target. */ From f31c643fb8faaa2af6e547c11cc70d067c7673e2 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:43:37 +0300 Subject: [PATCH 47/52] Update modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java Co-authored-by: korlov42 --- .../api/src/main/java/org/apache/ignite/compute/JobTarget.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index 85745ca383f..f094ca0c3ed 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -84,6 +84,8 @@ static JobTarget anyNode(Set nodes) { /** * Creates a colocated job target for a specific table and key. * + *

This target determines that a job should be executed on the same node that hosts the data for a given key of provided table. + * * @param tableName Table name. * @param key Key. * @return Job target. From 18a4a7ba61e3103dfce9862aaca95f4eb71b6877 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:43:46 +0300 Subject: [PATCH 48/52] Update modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java Co-authored-by: korlov42 --- .../api/src/main/java/org/apache/ignite/compute/JobTarget.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java index f094ca0c3ed..958269430e9 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/JobTarget.java @@ -97,6 +97,8 @@ static JobTarget colocated(String tableName, Tuple key) { /** * Creates a colocated job target for a specific table and key with mapper. * + *

This target determines that a job should be executed on the same node that hosts the data for a given key of provided table. + * * @param tableName Table name. * @param key Key. * @return Job target. From 0dab8c8bdffd861cbab2bc2bcad0a69ede9b659f Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:44:16 +0300 Subject: [PATCH 49/52] Update modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java Co-authored-by: Vadim Pakhnushev <8614891+valepakh@users.noreply.github.com> --- .../apache/ignite/internal/client/compute/ClientCompute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index bcaeae90e9f..f1b9a266c56 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -134,7 +134,7 @@ public JobExecution submit(JobTarget target, JobDescriptor descriptor, Ob @Override public R execute(JobTarget target, JobDescriptor descriptor, Object... args) { - return sync(this.submit(target, descriptor, args).resultAsync()); + return sync(executeAsync(target, descriptor, args)); } private CompletableFuture doExecuteColocatedAsync( From 5cbf11422c7ce7cb307a75a5ee62a9f369ef06aa Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:47:32 +0300 Subject: [PATCH 50/52] Fix checkstyle --- .../apache/ignite/internal/client/compute/ClientCompute.java | 2 +- .../org/apache/ignite/internal/compute/IgniteComputeImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java index f1b9a266c56..629dacb61e9 100644 --- a/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java +++ b/modules/client/src/main/java/org/apache/ignite/internal/client/compute/ClientCompute.java @@ -35,6 +35,7 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; +import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.ColocatedJobTarget; import org.apache.ignite.compute.DeploymentUnit; import org.apache.ignite.compute.IgniteCompute; @@ -42,7 +43,6 @@ import org.apache.ignite.compute.JobExecution; import org.apache.ignite.compute.JobExecutionOptions; import org.apache.ignite.compute.JobTarget; -import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.client.ClientUtils; import org.apache.ignite.internal.client.PayloadInputChannel; diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java index c22b5797892..e91ada79795 100644 --- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java +++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/IgniteComputeImpl.java @@ -39,6 +39,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.ColocatedJobTarget; import org.apache.ignite.compute.ComputeException; import org.apache.ignite.compute.DeploymentUnit; @@ -49,7 +50,6 @@ import org.apache.ignite.compute.JobStatus; import org.apache.ignite.compute.JobTarget; import org.apache.ignite.compute.NodeNotFoundException; -import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.task.MapReduceJob; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.hlc.HybridClock; From 45afbbfca6b682a5111d21ba2113434d2d84b8e9 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 11:57:33 +0300 Subject: [PATCH 51/52] Fix checkstyle --- .../test/java/org/apache/ignite/client/fakes/FakeCompute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java index c874d1acea7..b491288cbae 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java +++ b/modules/client/src/test/java/org/apache/ignite/client/fakes/FakeCompute.java @@ -39,6 +39,7 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; import org.apache.ignite.Ignite; +import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.ColocatedJobTarget; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.compute.DeploymentUnit; @@ -49,7 +50,6 @@ import org.apache.ignite.compute.JobState; import org.apache.ignite.compute.JobStatus; import org.apache.ignite.compute.JobTarget; -import org.apache.ignite.compute.AnyNodeJobTarget; import org.apache.ignite.compute.task.TaskExecution; import org.apache.ignite.internal.compute.ComputeUtils; import org.apache.ignite.internal.compute.IgniteComputeInternal; From 553977f2e1277171a855c125b0cdbe47989cf959 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 20 Jun 2024 12:29:24 +0300 Subject: [PATCH 52/52] fix javadoc --- .../main/java/org/apache/ignite/compute/AnyNodeJobTarget.java | 2 +- .../java/org/apache/ignite/compute/ColocatedJobTarget.java | 2 +- .../src/test/java/org/apache/ignite/client/TestServer.java | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/api/src/main/java/org/apache/ignite/compute/AnyNodeJobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/AnyNodeJobTarget.java index beb834b78d5..2075c0a0f5b 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/AnyNodeJobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/AnyNodeJobTarget.java @@ -22,7 +22,7 @@ import org.apache.ignite.network.ClusterNode; /** - * Nodes-based job execution target. + * Any node execution target. Indicates any node from the provided set. */ public class AnyNodeJobTarget implements JobTarget { private final Set nodes; diff --git a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedJobTarget.java b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedJobTarget.java index 861e5b7c907..669452b99b6 100644 --- a/modules/api/src/main/java/org/apache/ignite/compute/ColocatedJobTarget.java +++ b/modules/api/src/main/java/org/apache/ignite/compute/ColocatedJobTarget.java @@ -23,7 +23,7 @@ import org.jetbrains.annotations.Nullable; /** - * Colocated job execution target. + * Colocated job execution target. Indicates a node that hosts the data for the specified key in the provided table. */ public class ColocatedJobTarget implements JobTarget { private final String tableName; diff --git a/modules/client/src/test/java/org/apache/ignite/client/TestServer.java b/modules/client/src/test/java/org/apache/ignite/client/TestServer.java index 036dc130cb6..3354db7facd 100644 --- a/modules/client/src/test/java/org/apache/ignite/client/TestServer.java +++ b/modules/client/src/test/java/org/apache/ignite/client/TestServer.java @@ -97,8 +97,6 @@ public class TestServer implements AutoCloseable { private final FakePlacementDriver placementDriver = new FakePlacementDriver(FakeInternalTable.PARTITIONS); - private final CmgMessagesFactory msgFactory = new CmgMessagesFactory(); - /** * Constructor. * @@ -220,7 +218,7 @@ public TestServer( assertThat(authenticationManager.startAsync(componentContext), willCompleteSuccessfully()); } - ClusterTag tag = msgFactory.clusterTag() + ClusterTag tag = new CmgMessagesFactory().clusterTag() .clusterName("Test Server") .clusterId(clusterId) .build();