From f136bfa6e08daf874695190766192ca668dec575 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 24 Jan 2017 20:51:16 -0500 Subject: [PATCH 01/50] Adds support for persistent actions A persistent action is a transport-like action that is using the cluster state instead of transport to start tasks. This allows persistent tasks to survive restart of executing nodes. A persistent action can be implemented by extending TransportPersistentAction. TransportPersistentAction will start the task by using PersistentActionService, which controls persistent tasks lifecycle. See TestPersistentActionPlugin for an example implementing a persistent action. Original commit: elastic/x-pack@5e83f1bfa38f83e058f37655eaab8ed4f349bad5 --- .../CompletionPersistentTaskAction.java | 179 +++++++ .../PersistentActionCoordinator.java | 401 +++++++++++++++ .../persistent/PersistentActionExecutor.java | 60 +++ .../persistent/PersistentActionRegistry.java | 104 ++++ .../persistent/PersistentActionRequest.java | 35 ++ .../persistent/PersistentActionResponse.java | 70 +++ .../persistent/PersistentActionService.java | 76 +++ .../persistent/PersistentTask.java | 55 +++ .../PersistentTaskClusterService.java | 245 +++++++++ .../persistent/PersistentTasksInProgress.java | 215 ++++++++ .../RemovePersistentTaskAction.java | 209 ++++++++ .../persistent/StartPersistentTaskAction.java | 178 +++++++ .../persistent/TransportPersistentAction.java | 116 +++++ .../persistent/package-info.java | 48 ++ .../CancelPersistentTaskRequestTests.java | 35 ++ .../CancelPersistentTaskResponseTests.java | 35 ++ .../PersistentActionCoordinatorTests.java | 381 ++++++++++++++ .../persistent/PersistentActionIT.java | 180 +++++++ .../PersistentActionRegistryTests.java | 64 +++ .../PersistentActionResponseTests.java | 34 ++ .../PersistentTasksInProgressTests.java | 56 +++ .../RestartPersistentTaskRequestTests.java | 35 ++ .../StartPersistentActionRequestTests.java | 58 +++ .../TestPersistentActionPlugin.java | 466 ++++++++++++++++++ 24 files changed, 3335 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionResponse.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentTask.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java create mode 100644 server/src/main/java/org/elasticsearch/persistent/package-info.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentActionResponseTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java new file mode 100644 index 0000000000000..ec31c7247037b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -0,0 +1,179 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +/** + * Action that is used by executor node to indicate that the persistent action finished or failed on the node and needs to be + * removed from the cluster state in case of successful completion or restarted on some other node in case of failure. + */ +public class CompletionPersistentTaskAction extends Action { + + public static final CompletionPersistentTaskAction INSTANCE = new CompletionPersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/completion"; + + private CompletionPersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeRequest { + + private long taskId; + + private Exception exception; + + public Request() { + + } + + public Request(long taskId, Exception exception) { + this.taskId = taskId; + this.exception = exception; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + exception = in.readException(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + out.writeException(exception); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return taskId == request.taskId && + Objects.equals(exception, request.exception); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, exception); + } + } + + public static class Response extends AcknowledgedResponse { + + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, CompletionPersistentTaskAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener() { + @Override + public void onResponse(Empty empty) { + listener.onResponse(newResponse()); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java new file mode 100644 index 0000000000000..efb73471e1e4e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java @@ -0,0 +1,401 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Provider; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.transport.TransportResponse.Empty; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static java.util.Objects.requireNonNull; + +/** + * This component is responsible for coordination of execution of persistent actions on individual nodes. It runs on all + * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. + */ +public class PersistentActionCoordinator extends AbstractComponent implements ClusterStateListener { + private final Map runningTasks = new HashMap<>(); + private final PersistentActionService persistentActionService; + private final PersistentActionRegistry persistentActionRegistry; + private final TaskManager taskManager; + private final PersistentActionExecutor persistentActionExecutor; + + + public PersistentActionCoordinator(Settings settings, + PersistentActionService persistentActionService, + PersistentActionRegistry persistentActionRegistry, + TaskManager taskManager, + PersistentActionExecutor persistentActionExecutor) { + super(settings); + this.persistentActionService = persistentActionService; + this.persistentActionRegistry = persistentActionRegistry; + this.taskManager = taskManager; + this.persistentActionExecutor = persistentActionExecutor; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + PersistentTasksInProgress tasks = event.state().custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress previousTasks = event.previousState().custom(PersistentTasksInProgress.TYPE); + + if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { + // We have some changes let's check if they are related to our node + String localNodeId = event.state().getNodes().getLocalNodeId(); + Set notVisitedTasks = new HashSet<>(runningTasks.keySet()); + if (tasks != null) { + for (PersistentTaskInProgress taskInProgress : tasks.entries()) { + if (localNodeId.equals(taskInProgress.getExecutorNode())) { + PersistentTaskId persistentTaskId = new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()); + RunningPersistentTask persistentTask = runningTasks.get(persistentTaskId); + if (persistentTask == null) { + // New task - let's start it + startTask(taskInProgress); + } else { + // The task is still running + notVisitedTasks.remove(persistentTaskId); + if (persistentTask.getState() == State.FAILED_NOTIFICATION) { + // We tried to notify the master about this task before but the notification failed and + // the master doesn't seem to know about it - retry notification + restartCompletionNotification(persistentTask); + } + } + } + } + } + + for (PersistentTaskId id : notVisitedTasks) { + RunningPersistentTask task = runningTasks.get(id); + if (task.getState() == State.NOTIFIED || task.getState() == State.FAILED) { + // Result was sent to the caller and the caller acknowledged acceptance of the result + finishTask(id); + } else if (task.getState() == State.FAILED_NOTIFICATION) { + // We tried to send result to master, but it failed and master doesn't know about this task + // this shouldn't really happen, unless this node is severally out of sync with the master + logger.warn("failed to notify master about task {}", task.getId()); + finishTask(id); + } else { + // task is running locally, but master doesn't know about it - that means that the persistent task was removed + // cancel the task without notifying master + cancelTask(id); + } + } + + } + + } + + private void startTask(PersistentTaskInProgress taskInProgress) { + PersistentActionRegistry.PersistentActionHolder holder = + persistentActionRegistry.getPersistentActionHolderSafe(taskInProgress.getAction()); + PersistentTask task = (PersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]", + taskInProgress.getRequest()); + boolean processed = false; + try { + RunningPersistentTask runningPersistentTask = new RunningPersistentTask(task, taskInProgress.getId()); + task.setStatusProvider(runningPersistentTask); + PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask); + try { + runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask); + persistentActionExecutor.executeAction(taskInProgress.getRequest(), task, holder, listener); + } catch (Exception e) { + // Submit task failure + listener.onFailure(e); + } + processed = true; + } finally { + if (processed == false) { + // something went wrong - unregistering task + taskManager.unregister(task); + } + } + } + + private void finishTask(PersistentTaskId persistentTaskId) { + RunningPersistentTask task = runningTasks.remove(persistentTaskId); + if (task != null && task.getTask() != null) { + taskManager.unregister(task.getTask()); + } + } + + private void cancelTask(PersistentTaskId persistentTaskId) { + RunningPersistentTask task = runningTasks.remove(persistentTaskId); + if (task != null && task.getTask() != null) { + if (task.markAsCancelled()) { + persistentActionService.sendCancellation(task.getTask().getId(), new ActionListener() { + @Override + public void onResponse(CancelTasksResponse cancelTasksResponse) { + } + + @Override + public void onFailure(Exception e) { + // There is really nothing we can do in case of failure here + logger.warn((Supplier) () -> new ParameterizedMessage("failed to cancel task {}", task.getId()), e); + } + }); + } + } + } + + + private void restartCompletionNotification(RunningPersistentTask task) { + logger.trace("resending notification for task {}", task.getId()); + if (task.getState() == State.CANCELLED) { + taskManager.unregister(task.getTask()); + } else { + if (task.restartCompletionNotification()) { + persistentActionService.sendCompletionNotification(task.getId(), task.getFailure(), new PublishedResponseListener(task)); + } else { + logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState()); + } + } + } + + private void startCompletionNotification(RunningPersistentTask task, Exception e) { + if (task.getState() == State.CANCELLED) { + taskManager.unregister(task.getTask()); + } else { + logger.trace("sending notification for failed task {}", task.getId()); + if (task.startNotification(e)) { + persistentActionService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); + } else { + logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState()); + } + } + } + + private class PersistentTaskListener implements ActionListener { + private final RunningPersistentTask task; + + public PersistentTaskListener(final RunningPersistentTask task) { + this.task = task; + } + + @Override + public void onResponse(Empty response) { + startCompletionNotification(task, null); + } + + @Override + public void onFailure(Exception e) { + if (task.getTask().isCancelled()) { + // The task was explicitly cancelled - no need to restart it, just log the exception if it's not TaskCancelledException + if (e instanceof TaskCancelledException == false) { + logger.warn((Supplier) () -> new ParameterizedMessage( + "cancelled task {} failed with an exception, cancellation reason [{}]", + task.getId(), task.getTask().getReasonCancelled()), e); + } + startCompletionNotification(task, null); + } else { + startCompletionNotification(task, e); + } + } + } + + private class PublishedResponseListener implements ActionListener { + private final RunningPersistentTask task; + + public PublishedResponseListener(final RunningPersistentTask task) { + this.task = task; + } + + + @Override + public void onResponse(CompletionPersistentTaskAction.Response response) { + logger.trace("notification for task {} was successful", task.getId()); + if (task.markAsNotified() == false) { + logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getId(), task.getState()); + } + taskManager.unregister(task.getTask()); + } + + @Override + public void onFailure(Exception e) { + logger.warn((Supplier) () -> new ParameterizedMessage("notification for task {} failed - retrying", task.getId()), e); + if (task.notificationFailed() == false) { + logger.warn("attempt to mark restart notification for task {} in the {} state failed", task.getId(), task.getState()); + } + } + } + + public enum State { + STARTED, // the task is currently running + CANCELLED, // the task is cancelled + FAILED, // the task is done running and trying to notify caller + FAILED_NOTIFICATION, // the caller notification failed + NOTIFIED // the caller was notified, the task can be removed + } + + private static class PersistentTaskId { + private final long id; + private final long allocationId; + + public PersistentTaskId(long id, long allocationId) { + this.id = id; + this.allocationId = allocationId; + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTaskId that = (PersistentTaskId) o; + return id == that.id && + allocationId == that.allocationId; + } + + @Override + public int hashCode() { + return Objects.hash(id, allocationId); + } + } + + private static class RunningPersistentTask implements Provider { + private final PersistentTask task; + private final long id; + private final AtomicReference state; + @Nullable + private Exception failure; + + public RunningPersistentTask(PersistentTask task, long id) { + this(task, id, State.STARTED); + } + + public RunningPersistentTask(PersistentTask task, long id, State state) { + this.task = task; + this.id = id; + this.state = new AtomicReference<>(state); + } + + public PersistentTask getTask() { + return task; + } + + public long getId() { + return id; + } + + public State getState() { + return state.get(); + } + + public Exception getFailure() { + return failure; + } + + public boolean startNotification(Exception failure) { + boolean result = state.compareAndSet(State.STARTED, State.FAILED); + if (result) { + this.failure = failure; + } + return result; + } + + public boolean notificationFailed() { + return state.compareAndSet(State.FAILED, State.FAILED_NOTIFICATION); + } + + public boolean restartCompletionNotification() { + return state.compareAndSet(State.FAILED_NOTIFICATION, State.FAILED); + } + + public boolean markAsNotified() { + return state.compareAndSet(State.FAILED, State.NOTIFIED); + } + + public boolean markAsCancelled() { + return state.compareAndSet(State.STARTED, State.CANCELLED); + } + + @Override + public Task.Status get() { + return new Status(state.get()); + } + } + + public static class Status implements Task.Status { + public static final String NAME = "persistent_executor"; + + private final State state; + + public Status(State state) { + this.state = requireNonNull(state, "State cannot be null"); + } + + public Status(StreamInput in) throws IOException { + state = State.valueOf(in.readString()); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("state", state.toString()); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(state.toString()); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + public State getState() { + return state; + } + + @Override + public boolean isFragment() { + return false; + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java new file mode 100644 index 0000000000000..de6c6ebd0680d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; + +/** + * This component is responsible for execution of persistent actions. + */ +public class PersistentActionExecutor { + private final ThreadPool threadPool; + + public PersistentActionExecutor(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void executeAction(Request request, + PersistentTask task, + PersistentActionRegistry.PersistentActionHolder holder, + ActionListener listener) { + threadPool.executor(holder.getExecutor()).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @SuppressWarnings("unchecked") + @Override + protected void doRun() throws Exception { + try { + holder.getPersistentAction().nodeOperation(task, request, listener); + } catch (Exception ex) { + listener.onFailure(ex); + } + + } + }); + + } + +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java new file mode 100644 index 0000000000000..c0e9697a284f1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; + +import java.util.Collections; +import java.util.Map; + +/** + * Components that registers all persistent actions + */ +public class PersistentActionRegistry extends AbstractComponent { + + private volatile Map> actions = Collections.emptyMap(); + + private final Object actionHandlerMutex = new Object(); + + public PersistentActionRegistry(Settings settings) { + super(settings); + } + + public void registerPersistentAction(String action, + TransportPersistentAction persistentAction) { + registerPersistentAction(new PersistentActionHolder<>(action, persistentAction, persistentAction.getExecutor())); + } + + private void registerPersistentAction( + PersistentActionHolder reg) { + + synchronized (actionHandlerMutex) { + PersistentActionHolder replaced = actions.get(reg.getAction()); + actions = MapBuilder.newMapBuilder(actions).put(reg.getAction(), reg).immutableMap(); + if (replaced != null) { + logger.warn("registered two handlers for persistent action {}, handlers: {}, {}", reg.getAction(), reg, replaced); + } + } + } + + public void removeHandler(String action) { + synchronized (actionHandlerMutex) { + actions = MapBuilder.newMapBuilder(actions).remove(action).immutableMap(); + } + } + + @SuppressWarnings("unchecked") + public PersistentActionHolder getPersistentActionHolderSafe(String action) { + PersistentActionHolder holder = (PersistentActionHolder) actions.get(action); + if (holder == null) { + throw new IllegalStateException("Unknown persistent action [" + action + "]"); + } + return holder; + } + + public + TransportPersistentAction getPersistentActionSafe(String action) { + PersistentActionHolder holder = getPersistentActionHolderSafe(action); + return holder.getPersistentAction(); + } + + public static final class PersistentActionHolder { + + private final String action; + private final TransportPersistentAction persistentAction; + private final String executor; + + + public PersistentActionHolder(String action, TransportPersistentAction persistentAction, String executor) { + this.action = action; + this.persistentAction = persistentAction; + this.executor = executor; + } + + public String getAction() { + return action; + } + + public TransportPersistentAction getPersistentAction() { + return persistentAction; + } + + public String getExecutor() { + return executor; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java new file mode 100644 index 0000000000000..d1e6bcfe6ec20 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +/** + * Base class for a request for a persistent action + */ +public abstract class PersistentActionRequest extends ActionRequest implements NamedWriteable, ToXContent { + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return new PersistentTask(id, type, action, getDescription(), parentTaskId); + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionResponse.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionResponse.java new file mode 100644 index 0000000000000..69362198975cb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionResponse.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +/** + * Response upon a successful start or an persistent action + */ +public class PersistentActionResponse extends ActionResponse { + private long taskId; + + public PersistentActionResponse() { + super(); + } + + public PersistentActionResponse(long taskId) { + this.taskId = taskId; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + } + + public long getTaskId() { + return taskId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentActionResponse that = (PersistentActionResponse) o; + return taskId == that.taskId; + } + + @Override + public int hashCode() { + return Objects.hash(taskId); + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java new file mode 100644 index 0000000000000..7534b15e6343a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.TaskId; + +/** + * Service responsible for executing restartable actions that can survive disappearance of a coordinating and executor nodes. + */ +public class PersistentActionService extends AbstractComponent { + + private final Client client; + private final ClusterService clusterService; + + public PersistentActionService(Settings settings, ClusterService clusterService, Client client) { + super(settings); + this.client = client; + this.clusterService = clusterService; + } + + public void sendRequest(String action, Request request, + ActionListener listener) { + StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(action, request); + try { + client.execute(StartPersistentTaskAction.INSTANCE, startRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void sendCompletionNotification(long taskId, Exception failure, + ActionListener listener) { + CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); + try { + client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void sendCancellation(long taskId, ActionListener listener) { + DiscoveryNode localNode = clusterService.localNode(); + CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); + cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); + cancelTasksRequest.setReason("persistent action was removed"); + try { + client.admin().cluster().cancelTasks(cancelTasksRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java new file mode 100644 index 0000000000000..b122e8cda84dc --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.inject.Provider; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +/** + * Task that returns additional state information + */ +public class PersistentTask extends CancellableTask { + private Provider statusProvider; + + public PersistentTask(long id, String type, String action, String description, TaskId parentTask) { + super(id, type, action, description, parentTask); + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + + @Override + public Status getStatus() { + Provider statusProvider = this.statusProvider; + if (statusProvider != null) { + return statusProvider.get(); + } else { + return null; + } + } + + public void setStatusProvider(Provider statusProvider) { + assert this.statusProvider == null; + this.statusProvider = statusProvider; + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java new file mode 100644 index 0000000000000..4a76545bc8331 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java @@ -0,0 +1,245 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.transport.TransportResponse.Empty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Component that runs only on the master node and is responsible for assigning running tasks to nodes + */ +public class PersistentTaskClusterService extends AbstractComponent implements ClusterStateListener { + + private final ClusterService clusterService; + private final PersistentActionRegistry registry; + + public PersistentTaskClusterService(Settings settings, PersistentActionRegistry registry, ClusterService clusterService) { + super(settings); + this.clusterService = clusterService; + clusterService.addListener(this); + this.registry = registry; + + } + + /** + * Creates a new persistent task on master node + * + * @param action the action name + * @param request request + * @param listener the listener that will be called when task is started + */ + public void createPersistentTask(String action, Request request, + ActionListener listener) { + clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + final String executorNodeId = executorNode(action, currentState, request); + PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); + final List> currentTasks = new ArrayList<>(); + final long nextId; + if (tasksInProgress != null) { + nextId = tasksInProgress.getCurrentId() + 1; + currentTasks.addAll(tasksInProgress.entries()); + } else { + nextId = 1; + } + currentTasks.add(new PersistentTaskInProgress<>(nextId, action, request, executorNodeId)); + ClusterState.Builder builder = ClusterState.builder(currentState); + return builder.putCustom(PersistentTasksInProgress.TYPE, new PersistentTasksInProgress(nextId, currentTasks)).build(); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(((PersistentTasksInProgress) newState.custom(PersistentTasksInProgress.TYPE)).getCurrentId()); + } + }); + } + + + /** + * Restarts a record about a running persistent task from cluster state + * + * @param id the id of a persistent task + * @param failure the reason for restarting the task or null if the task completed successfully + * @param listener the listener that will be called when task is removed + */ + public void completeOrRestartPersistentTask(long id, Exception failure, ActionListener listener) { + final String source; + if (failure != null) { + logger.warn("persistent task " + id + " failed, restarting", failure); + source = "restart persistent task"; + } else { + source = "finish persistent task"; + } + clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); + if (tasksInProgress == null) { + // Nothing to do, the task was already deleted + return currentState; + } + + boolean found = false; + final List> currentTasks = new ArrayList<>(); + for (PersistentTaskInProgress taskInProgress : tasksInProgress.entries()) { + if (taskInProgress.getId() == id) { + assert found == false; + found = true; + if (failure != null) { + // If the task failed - we need to restart it on another node, otherwise we just remove it + String executorNode = executorNode(taskInProgress.getAction(), currentState, taskInProgress.getRequest()); + currentTasks.add(new PersistentTaskInProgress<>(taskInProgress, executorNode)); + } + } else { + currentTasks.add(taskInProgress); + } + } + if (found) { + ClusterState.Builder builder = ClusterState.builder(currentState); + PersistentTasksInProgress tasks = new PersistentTasksInProgress(tasksInProgress.getCurrentId(), currentTasks); + return builder.putCustom(PersistentTasksInProgress.TYPE, tasks).build(); + } else { + return currentState; + } + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(Empty.INSTANCE); + } + }); + } + + private String executorNode(String action, ClusterState currentState, Request request) { + TransportPersistentAction persistentAction = registry.getPersistentActionSafe(action); + persistentAction.validate(request, currentState); + DiscoveryNode executorNode = persistentAction.executorNode(request, currentState); + final String executorNodeId; + if (executorNode == null) { + // The executor node not available yet, we will create task with empty executor node and try + // again later + executorNodeId = null; + } else { + executorNodeId = executorNode.getId(); + } + return executorNodeId; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeMaster()) { + PersistentTasksInProgress tasks = event.state().custom(PersistentTasksInProgress.TYPE); + if (tasks != null && (event.nodesChanged() || event.previousState().nodes().isLocalNodeElectedMaster() == false)) { + // We need to check if removed nodes were running any of the tasks and reassign them + boolean reassignmentRequired = false; + Set removedNodes = event.nodesDelta().removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); + for (PersistentTaskInProgress taskInProgress : tasks.entries()) { + if (taskInProgress.getExecutorNode() == null) { + // there is an unassigned task - we need to try assigning it + reassignmentRequired = true; + break; + } + if (removedNodes.contains(taskInProgress.getExecutorNode())) { + // The caller node disappeared, we need to assign a new caller node + reassignmentRequired = true; + break; + } + } + if (reassignmentRequired) { + reassignTasks(); + } + } + } + } + + /** + * Evaluates the cluster state and tries to assign tasks to nodes + */ + public void reassignTasks() { + clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress tasks = currentState.custom(PersistentTasksInProgress.TYPE); + ClusterState newClusterState = currentState; + DiscoveryNodes nodes = currentState.nodes(); + if (tasks != null) { + // We need to check if removed nodes were running any of the tasks and reassign them + for (PersistentTaskInProgress task : tasks.entries()) { + if (task.getExecutorNode() == null || nodes.nodeExists(task.getExecutorNode()) == false) { + // there is an unassigned task - we need to try assigning it + String executorNode = executorNode(task.getAction(), currentState, task.getRequest()); + if (Objects.equals(executorNode, task.getExecutorNode()) == false) { + PersistentTasksInProgress tasksInProgress = newClusterState.custom(PersistentTasksInProgress.TYPE); + final List> currentTasks = new ArrayList<>(); + for (PersistentTaskInProgress taskInProgress : tasksInProgress.entries()) { + if (task.getId() == taskInProgress.getId()) { + currentTasks.add(new PersistentTaskInProgress<>(task, executorNode)); + } else { + currentTasks.add(taskInProgress); + } + } + newClusterState = ClusterState.builder(newClusterState).putCustom(PersistentTasksInProgress.TYPE, + new PersistentTasksInProgress(tasksInProgress.getCurrentId(), currentTasks)).build(); + } + } + } + } + return newClusterState; + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn("Unsuccessful persistent task reassignment", e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + + } + }); + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java new file mode 100644 index 0000000000000..07ea32ed10004 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -0,0 +1,215 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * A cluster state record that contains a list of all running persistent tasks + */ +public final class PersistentTasksInProgress extends AbstractNamedDiffable implements ClusterState.Custom { + public static final String TYPE = "persistent_tasks"; + + // TODO: Implement custom Diff for entries + private final List> entries; + + private final long currentId; + + public PersistentTasksInProgress(long currentId, List> entries) { + this.currentId = currentId; + this.entries = entries; + } + + public List> entries() { + return this.entries; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTasksInProgress that = (PersistentTasksInProgress) o; + return currentId == that.currentId && + Objects.equals(entries, that.entries); + } + + @Override + public int hashCode() { + return Objects.hash(entries, currentId); + } + + public long getNumberOfTasksOnNode(String nodeId, String action) { + return entries.stream().filter(task -> action.equals(task.action) && nodeId.equals(task.executorNode)).count(); + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_3_0_UNRELEASED; + } + + /** + * A record that represents a single running persistent task + */ + public static class PersistentTaskInProgress implements Writeable { + private final long id; + private final long allocationId; + private final String action; + private final Request request; + @Nullable + private final String executorNode; + + + public PersistentTaskInProgress(long id, String action, Request request, String executorNode) { + this(id, 0L, action, request, executorNode); + } + + public PersistentTaskInProgress(PersistentTaskInProgress persistentTaskInProgress, String newExecutorNode) { + this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId + 1L, + persistentTaskInProgress.action, persistentTaskInProgress.request, newExecutorNode); + } + + private PersistentTaskInProgress(long id, long allocationId, String action, Request request, String executorNode) { + this.id = id; + this.allocationId = allocationId; + this.action = action; + this.request = request; + this.executorNode = executorNode; + // Update parent request for starting tasks with correct parent task ID + request.setParentTask("cluster", id); + } + + @SuppressWarnings("unchecked") + private PersistentTaskInProgress(StreamInput in) throws IOException { + id = in.readLong(); + allocationId = in.readLong(); + action = in.readString(); + request = (Request) in.readNamedWriteable(PersistentActionRequest.class); + executorNode = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(id); + out.writeLong(allocationId); + out.writeString(action); + out.writeNamedWriteable(request); + out.writeOptionalString(executorNode); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTaskInProgress that = (PersistentTaskInProgress) o; + return id == that.id && + allocationId == that.allocationId && + Objects.equals(action, that.action) && + Objects.equals(request, that.request) && + Objects.equals(executorNode, that.executorNode); + } + + @Override + public int hashCode() { + return Objects.hash(id, allocationId, action, request, executorNode); + } + + public long getId() { + return id; + } + + public long getAllocationId() { + return allocationId; + } + + public String getAction() { + return action; + } + + public Request getRequest() { + return request; + } + + @Nullable + public String getExecutorNode() { + return executorNode; + } + + } + + @Override + public String getWriteableName() { + return TYPE; + } + + public PersistentTasksInProgress(StreamInput in) throws IOException { + currentId = in.readLong(); + entries = in.readList(PersistentTaskInProgress::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(currentId); + out.writeList(entries); + } + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(ClusterState.Custom.class, TYPE, in); + } + + public long getCurrentId() { + return currentId; + } + + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.field("current_id", currentId); + builder.startArray("running_tasks"); + for (PersistentTaskInProgress entry : entries) { + toXContent(entry, builder, params); + } + builder.endArray(); + return builder; + } + + public void toXContent(PersistentTaskInProgress entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + { + builder.field("uuid", entry.id); + builder.field("action", entry.action); + builder.field("request"); + entry.request.toXContent(builder, params); + builder.field("executor_node", entry.executorNode); + } + builder.endObject(); + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java new file mode 100644 index 0000000000000..fe65e9fd9df92 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -0,0 +1,209 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +public class RemovePersistentTaskAction extends Action { + + public static final RemovePersistentTaskAction INSTANCE = new RemovePersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/remove"; + + private RemovePersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeRequest { + + private long taskId; + + public Request() { + + } + + public Request(long taskId) { + this.taskId = taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return taskId == request.taskId; + } + + @Override + public int hashCode() { + return Objects.hash(taskId); + } + } + + public static class Response extends AcknowledgedResponse { + protected Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeAcknowledged(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AcknowledgedResponse that = (AcknowledgedResponse) o; + return isAcknowledged() == that.isAcknowledged(); + } + + @Override + public int hashCode() { + return Objects.hash(isAcknowledged()); + } + + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, RemovePersistentTaskAction action) { + super(client, action, new Request()); + } + + public final RequestBuilder setTaskId(long taskId) { + request.setTaskId(taskId); + return this; + } + + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, null, new ActionListener() { + @Override + public void onResponse(Empty empty) { + listener.onResponse(new Response(true)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java new file mode 100644 index 0000000000000..1ee89b097ba6c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -0,0 +1,178 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +/** + * Internal action used by TransportPersistentAction to add the record for the persistent action to the cluster state. + */ +public class StartPersistentTaskAction extends Action { + + public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/start"; + + private StartPersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + public static class Request extends MasterNodeRequest { + + private String action; + + private PersistentActionRequest request; + + public Request() { + + } + + public Request(String action, PersistentActionRequest request) { + this.action = action; + this.request = request; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + action = in.readString(); + request = in.readOptionalNamedWriteable(PersistentActionRequest.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(action); + out.writeOptionalNamedWriteable(request); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request1 = (Request) o; + return Objects.equals(action, request1.action) && + Objects.equals(request, request1.request); + } + + @Override + public int hashCode() { + return Objects.hash(action, request); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, StartPersistentTaskAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + PersistentActionRegistry persistentActionRegistry, + PersistentActionService persistentActionService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + PersistentActionExecutor executor = new PersistentActionExecutor(threadPool); + clusterService.addListener(new PersistentActionCoordinator(settings, persistentActionService, persistentActionRegistry, + transportService.getTaskManager(), executor)); + } + + @Override + protected String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + protected PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTaskClusterService.createPersistentTask(request.action, request.request, new ActionListener() { + @Override + public void onResponse(Long newTaskId) { + listener.onResponse(new PersistentActionResponse(newTaskId)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java new file mode 100644 index 0000000000000..e4fe4eb316da1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** + * An action that can survive restart of requesting or executing node. + * These actions are using cluster state rather than only transport service to send requests and responses. + */ +public abstract class TransportPersistentAction + extends HandledTransportAction { + + private final String executor; + private final PersistentActionService persistentActionService; + + protected TransportPersistentAction(Settings settings, String actionName, boolean canTripCircuitBreaker, ThreadPool threadPool, + TransportService transportService, PersistentActionService persistentActionService, + PersistentActionRegistry persistentActionRegistry, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + Supplier requestSupplier, String executor) { + super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver, + requestSupplier); + this.executor = executor; + this.persistentActionService = persistentActionService; + persistentActionRegistry.registerPersistentAction(actionName, this); + } + + /** + * Returns the node id where the request has to be executed, + *

+ * The default implementation returns the least loaded data node + */ + public DiscoveryNode executorNode(Request request, ClusterState clusterState) { + return selectLeastLoadedNode(clusterState, DiscoveryNode::isDataNode); + } + + /** + * Finds the least loaded node that satisfies the selector criteria + */ + protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate selector) { + long minLoad = Long.MAX_VALUE; + DiscoveryNode minLoadedNode = null; + PersistentTasksInProgress persistentTasksInProgress = clusterState.custom(PersistentTasksInProgress.TYPE); + for (DiscoveryNode node : clusterState.getNodes()) { + if (selector.test(node)) { + if (persistentTasksInProgress == null) { + // We don't have any task running yet, pick the first available node + return node; + } + long numberOfTasks = persistentTasksInProgress.getNumberOfTasksOnNode(node.getId(), actionName); + if (minLoad > numberOfTasks) { + minLoad = numberOfTasks; + minLoadedNode = node; + } + } + } + return minLoadedNode; + } + + /** + * Checks the current cluster state for compatibility with the request + *

+ * Throws an exception if the supplied request cannot be executed on the cluster in the current state. + */ + public void validate(Request request, ClusterState clusterState) { + + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + persistentActionService.sendRequest(actionName, request, listener); + } + + /** + * This operation will be executed on the executor node. + * + * If nodeOperation throws an exception or triggers listener.onFailure() method, the task will be restarted, + * possibly on a different node. If listener.onResponse() is called, the task is considered to be successfully + * completed and will be removed from the cluster state and not restarted. + */ + protected abstract void nodeOperation(PersistentTask task, Request request, ActionListener listener); + + public String getExecutor() { + return executor; + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/package-info.java b/server/src/main/java/org/elasticsearch/persistent/package-info.java new file mode 100644 index 0000000000000..9994dedeab80b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/package-info.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The Persistent Actions are actions responsible for executing restartable actions that can survive disappearance of a + * coordinating and executor nodes. + *

+ * In order to be resilient to node restarts, the persistent actions are using the cluster state instead of a transport service to send + * requests and responses. The execution is done in six phases: + *

+ * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent action. This action is handled + * by the {@link org.elasticsearch.persistent.PersistentActionService}, which is using + * {@link org.elasticsearch.persistent.PersistentTaskClusterService} to update cluster state with the record about running persistent + * task. + *

+ * 2. The master node updates the {@link org.elasticsearch.persistent.PersistentTasksInProgress} in the cluster state to indicate that + * there is a new persistent action + * running in the system. + *

+ * 3. The {@link org.elasticsearch.persistent.PersistentActionCoordinator} running on every node in the cluster monitors changes in + * the cluster state and starts execution of all new actions assigned to the node it is running on. + *

+ * 4. If the action fails to start on the node, the {@link org.elasticsearch.persistent.PersistentActionCoordinator} uses the + * {@link org.elasticsearch.persistent.PersistentTasksInProgress} to notify the + * {@link org.elasticsearch.persistent.PersistentActionService}, which reassigns the action to another node in the cluster. + *

+ * 5. If action finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the + * cluster state. + *

+ * 6. The {@link org.elasticsearch.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent action. + */ +package org.elasticsearch.persistent; diff --git a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java new file mode 100644 index 0000000000000..db263428cba4c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.persistent.RemovePersistentTaskAction.Request; +import org.elasticsearch.test.AbstractStreamableTestCase; + +public class CancelPersistentTaskRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomLong()); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java new file mode 100644 index 0000000000000..a2db4fad92fea --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.persistent.RemovePersistentTaskAction.Response; +import org.elasticsearch.test.AbstractStreamableTestCase; + +public class CancelPersistentTaskResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + return new Response(randomBoolean()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java new file mode 100644 index 0000000000000..08aaf9dcbcd47 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java @@ -0,0 +1,381 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.CompletionPersistentTaskAction.Response; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PersistentActionCoordinatorTests extends ESTestCase { + + private ClusterService createClusterService() { + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + return new ClusterService(Settings.builder().put("cluster.name", "PersistentActionExecutorTests").build(), + clusterSettings, null, () -> new DiscoveryNode(UUIDs.randomBase64UUID(), buildNewFakeTransportAddress(), + Version.CURRENT)); + } + + private DiscoveryNodes createTestNodes(int nonLocalNodesCount, Settings settings) { + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); + nodes.add(DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), "this_node")); + for (int i = 0; i < nonLocalNodesCount; i++) { + nodes.add(new DiscoveryNode("other_node_" + i, buildNewFakeTransportAddress(), Version.CURRENT)); + } + nodes.localNodeId("this_node"); + return nodes.build(); + } + + public void testStartTask() throws Exception { + ClusterService clusterService = createClusterService(); + PersistentActionService persistentActionService = mock(PersistentActionService.class); + PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); + @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + registry.registerPersistentAction("test", action); + + int nonLocalNodesCount = randomInt(10); + MockExecutor executor = new MockExecutor(); + PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, + registry, new TaskManager(Settings.EMPTY), executor); + + ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) + .build(); + + List> tasks = new ArrayList<>(); + long taskId = randomLong(); + boolean added = false; + if (nonLocalNodesCount > 0) { + for (int i = 0; i < randomInt(5); i++) { + tasks.add(new PersistentTaskInProgress<>(taskId, "test_action", new TestRequest("other_" + i), + "other_node_" + randomInt(nonLocalNodesCount))); + taskId++; + if (added == false && randomBoolean()) { + added = true; + tasks.add(new PersistentTaskInProgress<>(taskId, "test", new TestRequest("this_param"), "this_node")); + taskId++; + } + } + } + + if (added == false) { + logger.info("No local node action was added"); + + } + ClusterState newClusterState = ClusterState.builder(state) + .putCustom(PersistentTasksInProgress.TYPE, new PersistentTasksInProgress(taskId, tasks)).build(); + + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + if (added) { + // Action for this node was added, let's make sure it was invoked + assertThat(executor.executions.size(), equalTo(1)); + + // Add task on some other node + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "some_other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action wasn't called again + assertThat(executor.executions.size(), equalTo(1)); + + // Start another task on this node + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest("this_param"), "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action was called this time + assertThat(executor.size(), equalTo(2)); + + // Finish both tasks + executor.get(0).listener.onFailure(new RuntimeException()); + executor.get(1).listener.onResponse(Empty.INSTANCE); + long failedTaskId = executor.get(0).task.getParentTaskId().getId(); + long finishedTaskId = executor.get(1).task.getParentTaskId().getId(); + executor.clear(); + + // Add task on some other node + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "some_other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action wasn't called again + assertThat(executor.size(), equalTo(0)); + + // Simulate reallocation of the failed task on the same node + state = newClusterState; + newClusterState = reallocateTask(state, failedTaskId, "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Simulate removal of the finished task + state = newClusterState; + newClusterState = removeTask(state, finishedTaskId); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action was only allocated on this node once + assertThat(executor.size(), equalTo(1)); + } + + } + + public void testTaskCancellation() { + ClusterService clusterService = createClusterService(); + AtomicLong capturedTaskId = new AtomicLong(); + AtomicReference> capturedListener = new AtomicReference<>(); + PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, null, null) { + @Override + public void sendCancellation(long taskId, ActionListener listener) { + capturedTaskId.set(taskId); + capturedListener.set(listener); + } + }; + PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); + @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + registry.registerPersistentAction("test", action); + + int nonLocalNodesCount = randomInt(10); + MockExecutor executor = new MockExecutor(); + TaskManager taskManager = new TaskManager(Settings.EMPTY); + PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, + registry, taskManager, executor); + + ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) + .build(); + + ClusterState newClusterState = state; + // Allocate first task + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Check the the task is know to the task manager + assertThat(taskManager.getTasks().size(), equalTo(1)); + Task runningTask = taskManager.getTasks().values().iterator().next(); + long persistentId = runningTask.getParentTaskId().getId(); + long localId = runningTask.getId(); + // Make sure it returns correct status + Task.Status status = runningTask.getStatus(); + assertThat(status.toString(), equalTo("{\"state\":\"STARTED\"}")); + + state = newClusterState; + // Relocate the task to some other node or remove it completely + if (randomBoolean()) { + newClusterState = reallocateTask(state, persistentId, "some_other_node"); + } else { + newClusterState = removeTask(state, persistentId); + } + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure it returns correct status + assertThat(taskManager.getTasks().size(), equalTo(1)); + assertThat(taskManager.getTasks().values().iterator().next().getStatus().toString(), equalTo("{\"state\":\"CANCELLED\"}")); + + + // That should trigger cancellation request + assertThat(capturedTaskId.get(), equalTo(localId)); + // Notify successful cancellation + capturedListener.get().onResponse(new CancelTasksResponse()); + + // finish or fail task + if (randomBoolean()) { + executor.get(0).listener.onResponse(Empty.INSTANCE); + } else { + executor.get(0).listener.onFailure(new IOException("test")); + } + + // Check the the task is now removed from task manager + assertThat(taskManager.getTasks().values(), empty()); + + } + + public void testNotificationFailure() { + ClusterService clusterService = createClusterService(); + AtomicLong capturedTaskId = new AtomicLong(-1L); + AtomicReference capturedException = new AtomicReference<>(); + AtomicReference> capturedListener = new AtomicReference<>(); + PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, clusterService, null) { + @Override + public void sendCompletionNotification(long taskId, Exception failure, ActionListener listener) { + capturedTaskId.set(taskId); + capturedException.set(failure); + capturedListener.set(listener); + } + }; + PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); + @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + registry.registerPersistentAction("test", action); + + int nonLocalNodesCount = randomInt(10); + MockExecutor executor = new MockExecutor(); + TaskManager taskManager = new TaskManager(Settings.EMPTY); + PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, + registry, taskManager, executor); + + ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) + .build(); + + ClusterState newClusterState = state; + // Allocate first task + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Fail the task + executor.get(0).listener.onFailure(new RuntimeException("test failure")); + + // Check that notification was sent + assertThat(capturedException.get().getMessage(), equalTo("test failure")); + capturedException.set(null); + + // Simulate failure to notify + capturedListener.get().onFailure(new IOException("simulated notification failure")); + + // Allocate another task + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Check that notification was sent again + assertThat(capturedException.get().getMessage(), equalTo("test failure")); + + // Check the the task is still known by the task manager + assertThat(taskManager.getTasks().size(), equalTo(1)); + long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); + + // This time acknowledge notification + capturedListener.get().onResponse(new Response()); + + // Reallocate failed task to another node + state = newClusterState; + newClusterState = reallocateTask(state, id, "other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Check the the task is now removed from task manager + assertThat(taskManager.getTasks().values(), empty()); + + } + + private ClusterState addTask(ClusterState state, String action, Request request, + String node) { + PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); + List> tasks = prevTasks == null ? new ArrayList<>() : new ArrayList<>(prevTasks.entries()); + tasks.add(new PersistentTaskInProgress<>(prevTasks == null ? 0 : prevTasks.getCurrentId(), action, request, node)); + return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, + new PersistentTasksInProgress(prevTasks == null ? 1 : prevTasks.getCurrentId() + 1, tasks)).build(); + } + + private ClusterState reallocateTask(ClusterState state, long taskId, String node) { + PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); + List> tasks = prevTasks == null ? new ArrayList<>() : new ArrayList<>(prevTasks.entries()); + for (int i = 0; i < tasks.size(); i++) { + if (tasks.get(i).getId() == taskId) { + tasks.set(i, new PersistentTaskInProgress<>(tasks.get(i), node)); + return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, + new PersistentTasksInProgress(prevTasks == null ? 1 : prevTasks.getCurrentId() + 1, tasks)).build(); + } + } + fail("didn't find task with id " + taskId); + return null; + } + + private ClusterState removeTask(ClusterState state, long taskId) { + PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); + List> tasks = prevTasks == null ? new ArrayList<>() : new ArrayList<>(prevTasks.entries()); + for (int i = 0; i < tasks.size(); i++) { + if (tasks.get(i).getId() == taskId) { + tasks.remove(i); + return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, + new PersistentTasksInProgress(prevTasks == null ? 1 : prevTasks.getCurrentId() + 1, tasks)).build(); + } + } + fail("didn't find task with id " + taskId); + return null; + } + + private class Execution { + private final PersistentActionRequest request; + private final PersistentTask task; + private final PersistentActionRegistry.PersistentActionHolder holder; + private final ActionListener listener; + + public Execution(PersistentActionRequest request, PersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, + ActionListener listener) { + this.request = request; + this.task = task; + this.holder = holder; + this.listener = listener; + } + } + + private class MockExecutor extends PersistentActionExecutor { + private List executions = new ArrayList<>(); + + public MockExecutor() { + super(null); + } + + @Override + public void executeAction(Request request, PersistentTask task, + PersistentActionRegistry.PersistentActionHolder holder, + ActionListener listener) { + executions.add(new Execution(request, task, holder, listener)); + } + + public Execution get(int i) { + return executions.get(i); + } + + public int size() { + return executions.size(); + } + + public void clear() { + executions.clear(); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java new file mode 100644 index 0000000000000..b4738ec5932a1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestTasksRequestBuilder; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2) +public class PersistentActionIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(TestPersistentActionPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @Override + protected Collection> getMockPlugins() { + return super.getMockPlugins(); + } + + protected boolean ignoreExternalCluster() { + return true; + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .build(); + } + + public void testPersistentActionRestart() throws Exception { + long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), + equalTo(1)); + }); + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + .get().getTasks().get(0); + logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); + // Verifying parent + assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(taskId)); + assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); + + logger.info("Failing the running task"); + // Fail the running task and make sure it restarts properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("fail").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + assertBusy(() -> { + // Wait for the task to restart + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + .getTasks(); + logger.info("Found {} tasks", tasks.size()); + assertThat(tasks.size(), equalTo(1)); + // Make sure that restarted task is different + assertThat(tasks.get(0).getTaskId(), not(equalTo(firstRunningTask.getTaskId()))); + }); + + logger.info("Removing persistent task with id {}", firstRunningTask.getId()); + // Remove the persistent task + assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + + logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); + assertBusy(() -> { + // Wait for the task to disappear completely + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), + empty()); + }); + } + + public void testPersistentActionCompletion() throws Exception { + long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), + equalTo(1)); + }); + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + .get().getTasks().get(0); + logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); + // Verifying parent + assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(taskId)); + assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); + + if (randomBoolean()) { + logger.info("Completing the running task"); + // Complete the running task and make sure it finishes properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + } else { + logger.info("Cancelling the running task"); + // Cancel the running task and make sure it finishes properly + assertThat(client().admin().cluster().prepareCancelTasks().setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + } + + assertBusy(() -> { + // Wait for the task to finish + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + .getTasks(); + logger.info("Found {} tasks", tasks.size()); + assertThat(tasks.size(), equalTo(0)); + + // Make sure the task is removed from the cluster state + assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE)) + .entries(), empty()); + }); + } + + public void testPersistentActionWithNoAvailableNode() throws Exception { + long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah") + .executorNodeAttr("test").get().getTaskId(); + + Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); + String newNode = internalCluster().startNode(nodeSettings); + String newNodeId = internalCluster().clusterService(newNode).localNode().getId(); + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), + equalTo(1)); + }); + TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + .get().getTasks().get(0); + + // Verifying the the task runs on the new node + assertThat(taskInfo.getTaskId().getNodeId(), equalTo(newNodeId)); + + internalCluster().stopRandomNode(settings -> "test".equals(settings.get("node.attr.test_attr"))); + + assertBusy(() -> { + // Wait for the task to disappear completely + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), + empty()); + }); + + // Remove the persistent task + assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + + } + + +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java new file mode 100644 index 0000000000000..efc48440cc971 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PersistentActionRegistryTests extends ESTestCase { + + public void testActionLookup() { + PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); + TransportPersistentAction action1 = mock(TransportPersistentAction.class); + when(action1.getExecutor()).thenReturn(ThreadPool.Names.MANAGEMENT); + TransportPersistentAction action2 = mock(TransportPersistentAction.class); + when(action2.getExecutor()).thenReturn(ThreadPool.Names.GENERIC); + registry.registerPersistentAction("test1", action1); + registry.registerPersistentAction("test2", action2); + + assertEquals(registry.getPersistentActionHolderSafe("test1").getAction(), "test1"); + assertEquals(registry.getPersistentActionHolderSafe("test1").getExecutor(), ThreadPool.Names.MANAGEMENT); + assertEquals(registry.getPersistentActionHolderSafe("test1").getPersistentAction(), action1); + assertEquals(registry.getPersistentActionSafe("test1"), action1); + + assertEquals(registry.getPersistentActionHolderSafe("test2").getAction(), "test2"); + assertEquals(registry.getPersistentActionHolderSafe("test2").getExecutor(), ThreadPool.Names.GENERIC); + assertEquals(registry.getPersistentActionHolderSafe("test2").getPersistentAction(), action2); + assertEquals(registry.getPersistentActionSafe("test2"), action2); + + try { + registry.getPersistentActionHolderSafe("test3"); + fail("Should have failed"); + } catch (IllegalStateException ex) { + assertEquals(ex.getMessage(), "Unknown persistent action [test3]"); + } + + try { + registry.getPersistentActionSafe("test3"); + fail("Should have failed"); + } catch (IllegalStateException ex) { + assertEquals(ex.getMessage(), "Unknown persistent action [test3]"); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionResponseTests.java new file mode 100644 index 0000000000000..2a1f24dabed40 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionResponseTests.java @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.test.AbstractStreamableTestCase; + +public class PersistentActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected PersistentActionResponse createTestInstance() { + return new PersistentActionResponse(randomLong()); + } + + @Override + protected PersistentActionResponse createBlankInstance() { + return new PersistentActionResponse(); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java new file mode 100644 index 0000000000000..4bdb6f7bc9c41 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class PersistentTasksInProgressTests extends AbstractWireSerializingTestCase { + + @Override + protected PersistentTasksInProgress createTestInstance() { + int numberOfTasks = randomInt(10); + List> entries = new ArrayList<>(); + for (int i = 0; i < numberOfTasks; i++) { + entries.add(new PersistentTasksInProgress.PersistentTaskInProgress<>( + randomLong(), randomAsciiOfLength(10), new TestPersistentActionPlugin.TestRequest(randomAsciiOfLength(10)), + randomAsciiOfLength(10))); + } + return new PersistentTasksInProgress(randomLong(), entries); + } + + @Override + protected Writeable.Reader instanceReader() { + return PersistentTasksInProgress::new; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.singletonList( + new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestPersistentActionPlugin.TestRequest::new) + )); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java new file mode 100644 index 0000000000000..59883dc8d4e40 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.persistent.CompletionPersistentTaskAction.Request; +import org.elasticsearch.test.AbstractStreamableTestCase; + +public class RestartPersistentTaskRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomLong(), null); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java new file mode 100644 index 0000000000000..52be69cbeb885 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.persistent.StartPersistentTaskAction.Request; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.util.Collections; + +public class StartPersistentActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + TestRequest testRequest = new TestRequest(); + if (randomBoolean()) { + testRequest.setTestParam(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + testRequest.setParentTask(randomAsciiOfLengthBetween(1, 20), randomLong()); + } + if (randomBoolean()) { + testRequest.setExecutorNodeAttr(randomAsciiOfLengthBetween(1, 20)); + } + return new Request(randomAsciiOfLengthBetween(1, 20), new TestRequest()); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.singletonList( + new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new) + )); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java new file mode 100644 index 0000000000000..28e7a6fca5264 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java @@ -0,0 +1,466 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.TaskOperationFailure; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksRequest; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.action.support.tasks.TasksRequestBuilder; +import org.elasticsearch.action.support.tasks.TransportTasksAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.test.ESTestCase.awaitBusy; +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * A plugin that adds a test persistent task. + */ +public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { + + @Override + public List> getActions() { + return Arrays.asList( + new ActionHandler<>(TestPersistentAction.INSTANCE, TransportTestPersistentAction.class), + new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), + new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), + new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), + new ActionHandler<>(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class) + ); + } + + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry) { + + PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, clusterService, client); + PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY); + return Arrays.asList( + persistentActionService, + persistentActionRegistry, + new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService) + ); + } + + @Override + public List getNamedWriteables() { + return Arrays.asList( + new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), + new NamedWriteableRegistry.Entry(PersistentActionCoordinator.Status.class, + PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), + new NamedWriteableRegistry.Entry(ClusterState.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom) + ); + } + + public static class TestRequest extends PersistentActionRequest { + + private String executorNodeAttr = null; + + private String responseNode = null; + + private String testParam = null; + + public TestRequest() { + + } + + public TestRequest(String testParam) { + this.testParam = testParam; + } + + public TestRequest(StreamInput in) throws IOException { + readFrom(in); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public String getWriteableName() { + return TestPersistentAction.NAME; + } + + public void setExecutorNodeAttr(String executorNodeAttr) { + this.executorNodeAttr = executorNodeAttr; + } + + public void setTestParam(String testParam) { + this.testParam = testParam; + } + + public String getExecutorNodeAttr() { + return executorNodeAttr; + } + + public String getTestParam() { + return testParam; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(executorNodeAttr); + out.writeOptionalString(responseNode); + out.writeOptionalString(testParam); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + executorNodeAttr = in.readOptionalString(); + responseNode = in.readOptionalString(); + testParam = in.readOptionalString(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestRequest that = (TestRequest) o; + return Objects.equals(executorNodeAttr, that.executorNodeAttr) && + Objects.equals(responseNode, that.responseNode) && + Objects.equals(testParam, that.testParam); + } + + @Override + public int hashCode() { + return Objects.hash(executorNodeAttr, responseNode, testParam); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return new TestTask(id, type, action, getDescription(), parentTaskId); + } + } + + public static class TestPersistentTaskRequestBuilder extends + ActionRequestBuilder { + + protected TestPersistentTaskRequestBuilder(ElasticsearchClient client, Action action, TestRequest request) { + super(client, action, request); + } + + public TestPersistentTaskRequestBuilder testParam(String testParam) { + request.setTestParam(testParam); + return this; + } + + public TestPersistentTaskRequestBuilder executorNodeAttr(String targetNode) { + request.setExecutorNodeAttr(targetNode); + return this; + } + + } + + public static class TestPersistentAction extends Action { + + public static final TestPersistentAction INSTANCE = new TestPersistentAction(); + public static final String NAME = "cluster:admin/persistent/test"; + + private TestPersistentAction() { + super(NAME); + } + + @Override + public PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + @Override + public TestPersistentTaskRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new TestPersistentTaskRequestBuilder(client, this, new TestRequest()); + } + } + + + public static class TransportTestPersistentAction extends TransportPersistentAction { + + private final TransportService transportService; + + @Inject + public TransportTestPersistentAction(Settings settings, ThreadPool threadPool, TransportService transportService, + PersistentActionService persistentActionService, + PersistentActionRegistry persistentActionRegistry, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, TestPersistentAction.NAME, false, threadPool, transportService, persistentActionService, + persistentActionRegistry, actionFilters, indexNameExpressionResolver, TestRequest::new, + ThreadPool.Names.MANAGEMENT); + this.transportService = transportService; + } + + @Override + public DiscoveryNode executorNode(TestRequest request, ClusterState clusterState) { + if (request.getExecutorNodeAttr() == null) { + return super.executorNode(request, clusterState); + } else { + return selectLeastLoadedNode(clusterState, + discoveryNode -> request.getExecutorNodeAttr().equals(discoveryNode.getAttributes().get("test_attr"))); + } + } + + @Override + protected void nodeOperation(PersistentTask task, TestRequest request, ActionListener listener) { + logger.info("started node operation for the task {}", task); + try { + TestTask testTask = (TestTask) task; + assertTrue(awaitBusy(() -> testTask.isCancelled() || + testTask.getOperation() != null || + transportService.lifecycleState() != Lifecycle.State.STARTED)); // speedup finishing on closed nodes + if (transportService.lifecycleState() == Lifecycle.State.STARTED) { + if ("finish".equals(testTask.getOperation())) { + listener.onResponse(Empty.INSTANCE); + } else if ("fail".equals(testTask.getOperation())) { + listener.onFailure(new RuntimeException("Simulating failure")); + } else if (testTask.isCancelled()) { + // Cancellation make cause different ways for the task to finish + if (randomBoolean()) { + if (randomBoolean()) { + listener.onFailure(new TaskCancelledException(testTask.getReasonCancelled())); + } else { + listener.onResponse(Empty.INSTANCE); + } + } else { + listener.onFailure(new RuntimeException(testTask.getReasonCancelled())); + } + } else { + fail("We really shouldn't be here"); + } + } + } catch (InterruptedException e) { + listener.onFailure(e); + } + } + } + + public static class TestTaskAction extends Action { + + public static final TestTaskAction INSTANCE = new TestTaskAction(); + public static final String NAME = "cluster:admin/persistent/task_test"; + + private TestTaskAction() { + super(NAME); + } + + @Override + public TestTasksResponse newResponse() { + return new TestTasksResponse(); + } + + @Override + public TestTasksRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new TestTasksRequestBuilder(client); + } + } + + + public static class TestTask extends PersistentTask { + private volatile String operation; + + public TestTask(long id, String type, String action, String description, TaskId parentTask) { + super(id, type, action, description, parentTask); + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + } + + static class TestTaskResponse implements Writeable { + + public TestTaskResponse() { + + } + + public TestTaskResponse(StreamInput in) throws IOException { + in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(true); + } + } + + public static class TestTasksRequest extends BaseTasksRequest { + private String operation; + + public TestTasksRequest() { + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + operation = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(operation); + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getOperation() { + return operation; + } + + } + + public static class TestTasksRequestBuilder extends TasksRequestBuilder { + + protected TestTasksRequestBuilder(ElasticsearchClient client) { + super(client, TestTaskAction.INSTANCE, new TestTasksRequest()); + } + + public TestTasksRequestBuilder setOperation(String operation) { + request.setOperation(operation); + return this; + } + } + + public static class TestTasksResponse extends BaseTasksResponse { + + private List tasks; + + public TestTasksResponse() { + + } + + public TestTasksResponse(List tasks, List taskFailures, + List nodeFailures) { + super(taskFailures, nodeFailures); + this.tasks = tasks == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(tasks)); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + tasks = in.readList(TestTaskResponse::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(tasks); + } + + public List getTasks() { + return tasks; + } + } + + public static class TransportTestTaskAction extends TransportTasksAction { + + @Inject + public TransportTestTaskAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, + TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, String nodeExecutor) { + super(settings, TestTaskAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, + TestTasksRequest::new, TestTasksResponse::new, ThreadPool.Names.MANAGEMENT); + } + + @Override + protected TestTasksResponse newResponse(TestTasksRequest request, List tasks, + List taskOperationFailures, + List failedNodeExceptions) { + return new TestTasksResponse(tasks, taskOperationFailures, failedNodeExceptions); + } + + @Override + protected TestTaskResponse readTaskResponse(StreamInput in) throws IOException { + return new TestTaskResponse(in); + } + + @Override + protected void taskOperation(TestTasksRequest request, TestTask task, ActionListener listener) { + task.setOperation(request.operation); + listener.onResponse(new TestTaskResponse()); + } + + @Override + protected boolean accumulateExceptions() { + return false; + } + } + + +} From 777b21f2ef874ed918a84a39ad8fe7bbe079fa0e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 30 Jan 2017 16:23:05 +0100 Subject: [PATCH 02/50] Add a number of auxiliary methods to persistent tasks classes. Original commit: elastic/x-pack@7f44b41b7a5b53a27a97d54c15ebbdf52bb00a87 --- .../persistent/PersistentTasksInProgress.java | 16 ++++++++++++++++ .../persistent/RemovePersistentTaskAction.java | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index 07ea32ed10004..982295c598daa 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -30,8 +30,11 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; /** * A cluster state record that contains a list of all running persistent tasks @@ -53,6 +56,19 @@ public List> entries() { return this.entries; } + public Collection> findEntries(String actionName, Predicate> predicate) { + return this.entries().stream() + .filter(p -> actionName.equals(p.getAction())) + .filter(predicate) + .collect(Collectors.toList()); + } + + public boolean entriesExist(String actionName, Predicate> predicate) { + return this.entries().stream() + .filter(p -> actionName.equals(p.getAction())) + .anyMatch(predicate); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java index fe65e9fd9df92..3cbd1f585279d 100644 --- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -112,7 +112,7 @@ public int hashCode() { } public static class Response extends AcknowledgedResponse { - protected Response() { + public Response() { super(); } From ac67d02bc3501281e809776a7df2087719f2c4ae Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 1 Feb 2017 18:06:22 -0500 Subject: [PATCH 03/50] Add support for task status on persistent tasks Similarly to task status on normal tasks it's now possible to update task status on the persistent tasks. This should allow updating the state of the running tasks (such as loading, started, etc) as well as store intermediate state or progress. Original commit: elastic/x-pack@048006b467421e45e6c76dc2578dc7995bc76426 --- .../PersistentActionCoordinator.java | 1 + .../persistent/PersistentActionService.java | 11 + .../persistent/PersistentTask.java | 10 + .../PersistentTaskClusterService.java | 60 ++++- .../persistent/PersistentTasksInProgress.java | 28 ++- .../persistent/TransportPersistentAction.java | 24 +- .../UpdatePersistentTaskStatusAction.java | 223 ++++++++++++++++++ .../persistent/PersistentActionIT.java | 66 +++++- .../PersistentTasksInProgressTests.java | 21 +- .../TestPersistentActionPlugin.java | 108 ++++++++- 10 files changed, 518 insertions(+), 34 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java index efb73471e1e4e..80faf8083bc41 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java @@ -132,6 +132,7 @@ private void startTask(PersistentTaskI try { RunningPersistentTask runningPersistentTask = new RunningPersistentTask(task, taskInProgress.getId()); task.setStatusProvider(runningPersistentTask); + task.setPersistentTaskId(taskInProgress.getId()); PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask); try { runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java index 7534b15e6343a..073dfb059aab4 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; /** @@ -73,4 +74,14 @@ public void sendCancellation(long taskId, ActionListener li listener.onFailure(e); } } + + public void updateStatus(long taskId, Task.Status status, ActionListener listener) { + UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status); + try { + client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java index b122e8cda84dc..559f970ba925b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java @@ -29,6 +29,8 @@ public class PersistentTask extends CancellableTask { private Provider statusProvider; + private long persistentTaskId; + public PersistentTask(long id, String type, String action, String description, TaskId parentTask) { super(id, type, action, description, parentTask); } @@ -52,4 +54,12 @@ public void setStatusProvider(Provider statusProvider) { assert this.statusProvider == null; this.statusProvider = statusProvider; } + + public long getPersistentTaskId() { + return persistentTaskId; + } + + public void setPersistentTaskId(long persistentTaskId) { + this.persistentTaskId = persistentTaskId; + } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java index 4a76545bc8331..53c8ecefb64ec 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import org.elasticsearch.transport.TransportResponse.Empty; @@ -133,13 +134,50 @@ public ClusterState execute(ClusterState currentState) throws Exception { currentTasks.add(taskInProgress); } } - if (found) { - ClusterState.Builder builder = ClusterState.builder(currentState); - PersistentTasksInProgress tasks = new PersistentTasksInProgress(tasksInProgress.getCurrentId(), currentTasks); - return builder.putCustom(PersistentTasksInProgress.TYPE, tasks).build(); - } else { + return rebuildClusterStateIfNeeded(found, currentState, currentTasks); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(Empty.INSTANCE); + } + }); + } + + /** + * Update task status + * + * @param id the id of a persistent task + * @param status new status + * @param listener the listener that will be called when task is removed + */ + public void updatePersistentTaskStatus(long id, Task.Status status, ActionListener listener) { + clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); + if (tasksInProgress == null) { + // Nothing to do, the task no longer exists return currentState; } + + boolean found = false; + final List> currentTasks = new ArrayList<>(); + for (PersistentTaskInProgress taskInProgress : tasksInProgress.entries()) { + if (taskInProgress.getId() == id) { + assert found == false; + found = true; + currentTasks.add(new PersistentTaskInProgress<>(taskInProgress, status)); + } else { + currentTasks.add(taskInProgress); + } + } + return rebuildClusterStateIfNeeded(found, currentState, currentTasks); } @Override @@ -154,6 +192,18 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } + private ClusterState rebuildClusterStateIfNeeded(boolean rebuild, ClusterState oldState, + List> currentTasks) { + if (rebuild) { + ClusterState.Builder builder = ClusterState.builder(oldState); + PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress tasks = new PersistentTasksInProgress(oldTasks.getCurrentId(), currentTasks); + return builder.putCustom(PersistentTasksInProgress.TYPE, tasks).build(); + } else { + return oldState; + } + } + private String executorNode(String action, ClusterState currentState, Request request) { TransportPersistentAction persistentAction = registry.getPersistentActionSafe(action); persistentAction.validate(request, currentState); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index 982295c598daa..cd29a8e484bf8 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -28,6 +28,8 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.Task.Status; import java.io.IOException; import java.util.Collection; @@ -101,23 +103,31 @@ public static class PersistentTaskInProgress persistentTaskInProgress, String newExecutorNode) { this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId + 1L, - persistentTaskInProgress.action, persistentTaskInProgress.request, newExecutorNode); + persistentTaskInProgress.action, persistentTaskInProgress.request, null, newExecutorNode); + } + + public PersistentTaskInProgress(PersistentTaskInProgress persistentTaskInProgress, Status status) { + this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId, + persistentTaskInProgress.action, persistentTaskInProgress.request, status, persistentTaskInProgress.executorNode); } - private PersistentTaskInProgress(long id, long allocationId, String action, Request request, String executorNode) { + private PersistentTaskInProgress(long id, long allocationId, String action, Request request, Status status, String executorNode) { this.id = id; this.allocationId = allocationId; this.action = action; this.request = request; + this.status = status; this.executorNode = executorNode; // Update parent request for starting tasks with correct parent task ID request.setParentTask("cluster", id); @@ -129,6 +139,7 @@ private PersistentTaskInProgress(StreamInput in) throws IOException { allocationId = in.readLong(); action = in.readString(); request = (Request) in.readNamedWriteable(PersistentActionRequest.class); + status = in.readOptionalNamedWriteable(Task.Status.class); executorNode = in.readOptionalString(); } @@ -138,6 +149,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(allocationId); out.writeString(action); out.writeNamedWriteable(request); + out.writeOptionalNamedWriteable(status); out.writeOptionalString(executorNode); } @@ -150,12 +162,13 @@ public boolean equals(Object o) { allocationId == that.allocationId && Objects.equals(action, that.action) && Objects.equals(request, that.request) && + Objects.equals(status, that.status) && Objects.equals(executorNode, that.executorNode); } @Override public int hashCode() { - return Objects.hash(id, allocationId, action, request, executorNode); + return Objects.hash(id, allocationId, action, request, status, executorNode); } public long getId() { @@ -179,6 +192,10 @@ public String getExecutorNode() { return executorNode; } + @Nullable + public Status getStatus() { + return status; + } } @Override @@ -224,6 +241,9 @@ public void toXContent(PersistentTaskInProgress entry, XContentBuilder builde builder.field("action", entry.action); builder.field("request"); entry.request.toXContent(builder, params); + if (entry.status != null) { + builder.field("status", entry.status, params); + } builder.field("executor_node", entry.executorNode); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java index e4fe4eb316da1..7e98262a6d6e2 100644 --- a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; @@ -101,9 +102,30 @@ protected void doExecute(Request request, ActionListener + * The status can be used to store the current progress of the task or provide an insight for the + * task allocator about the state of the currently running tasks. + */ + protected void updatePersistentTaskStatus(PersistentTask task, Task.Status status, ActionListener listener) { + persistentActionService.updateStatus(task.getPersistentTaskId(), status, + new ActionListener() { + @Override + public void onResponse(UpdatePersistentTaskStatusAction.Response response) { + listener.onResponse(Empty.INSTANCE); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + /** * This operation will be executed on the executor node. - * + *

* If nodeOperation throws an exception or triggers listener.onFailure() method, the task will be restarted, * possibly on a different node. If listener.onResponse() is called, the task is considered to be successfully * completed and will be removed from the cluster state and not restarted. diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java new file mode 100644 index 0000000000000..332dbceb28773 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -0,0 +1,223 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +public class UpdatePersistentTaskStatusAction extends Action { + + public static final UpdatePersistentTaskStatusAction INSTANCE = new UpdatePersistentTaskStatusAction(); + public static final String NAME = "cluster:admin/persistent/update_status"; + + private UpdatePersistentTaskStatusAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeRequest { + + private long taskId; + + private Task.Status status; + + public Request() { + + } + + public Request(long taskId, Task.Status status) { + this.taskId = taskId; + this.status = status; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + public void setStatus(Task.Status status) { + this.status = status; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + status = in.readOptionalNamedWriteable(Task.Status.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + out.writeOptionalNamedWriteable(status); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return taskId == request.taskId && + Objects.equals(status, request.status); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, status); + } + } + + public static class Response extends AcknowledgedResponse { + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeAcknowledged(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AcknowledgedResponse that = (AcknowledgedResponse) o; + return isAcknowledged() == that.isAcknowledged(); + } + + @Override + public int hashCode() { + return Objects.hash(isAcknowledged()); + } + + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, UpdatePersistentTaskStatusAction action) { + super(client, action, new Request()); + } + + public final RequestBuilder setTaskId(long taskId) { + request.setTaskId(taskId); + return this; + } + + public final RequestBuilder setStatus(Task.Status status) { + request.setStatus(status); + return this; + } + + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + persistentTaskClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener() { + @Override + public void onResponse(Empty empty) { + listener.onResponse(new Response(true)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java index b4738ec5932a1..2c1b7ccefb9b1 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java @@ -34,6 +34,8 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2) public class PersistentActionIT extends ESIntegTestCase { @@ -132,17 +134,7 @@ public void testPersistentActionCompletion() throws Exception { } - assertBusy(() -> { - // Wait for the task to finish - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() - .getTasks(); - logger.info("Found {} tasks", tasks.size()); - assertThat(tasks.size(), equalTo(0)); - - // Make sure the task is removed from the cluster state - assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE)) - .entries(), empty()); - }); + assertNoRunningTasks(); } public void testPersistentActionWithNoAvailableNode() throws Exception { @@ -176,5 +168,57 @@ public void testPersistentActionWithNoAvailableNode() throws Exception { } + public void testPersistentActionStatusUpdate() throws Exception { + TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get(); + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), + equalTo(1)); + }); + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + .get().getTasks().get(0); + + PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE); + assertThat(tasksInProgress.entries().size(), equalTo(1)); + assertThat(tasksInProgress.entries().get(0).getStatus(), nullValue()); + + int numberOfUpdates = randomIntBetween(1, 10); + for (int i = 0; i < numberOfUpdates; i++) { + logger.info("Updating the task status"); + // Complete the running task and make sure it finishes properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("update_status").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + int finalI = i; + assertBusy(() -> { + PersistentTasksInProgress tasks = internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE); + assertThat(tasks.entries().size(), equalTo(1)); + assertThat(tasks.entries().get(0).getStatus(), notNullValue()); + assertThat(tasks.entries().get(0).getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); + }); + + } + + logger.info("Completing the running task"); + // Complete the running task and make sure it finishes properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + assertNoRunningTasks(); + } + + private void assertNoRunningTasks() throws Exception { + assertBusy(() -> { + // Wait for the task to finish + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + .getTasks(); + logger.info("Found {} tasks", tasks.size()); + assertThat(tasks.size(), equalTo(0)); + + // Make sure the task is removed from the cluster state + assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE)) + .entries(), empty()); + }); + } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java index 4bdb6f7bc9c41..e9f91740507b6 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java @@ -21,11 +21,14 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.List; public class PersistentTasksInProgressTests extends AbstractWireSerializingTestCase { @@ -33,11 +36,16 @@ public class PersistentTasksInProgressTests extends AbstractWireSerializingTestC @Override protected PersistentTasksInProgress createTestInstance() { int numberOfTasks = randomInt(10); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); for (int i = 0; i < numberOfTasks; i++) { - entries.add(new PersistentTasksInProgress.PersistentTaskInProgress<>( + PersistentTaskInProgress taskInProgress = new PersistentTaskInProgress<>( randomLong(), randomAsciiOfLength(10), new TestPersistentActionPlugin.TestRequest(randomAsciiOfLength(10)), - randomAsciiOfLength(10))); + randomAsciiOfLength(10)); + if (randomBoolean()) { + // From time to time update status + taskInProgress = new PersistentTaskInProgress<>(taskInProgress, new Status(randomAsciiOfLength(10))); + } + entries.add(taskInProgress); } return new PersistentTasksInProgress(randomLong(), entries); } @@ -49,8 +57,9 @@ protected Writeable.Reader instanceReader() { @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(Collections.singletonList( - new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestPersistentActionPlugin.TestRequest::new) + return new NamedWriteableRegistry(Arrays.asList( + new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestPersistentActionPlugin.TestRequest::new), + new Entry(Task.Status.class, Status.NAME, Status::new) )); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java index 28e7a6fca5264..f0a1beaa46169 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java @@ -39,6 +39,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -55,6 +56,7 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; @@ -66,7 +68,12 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import static java.util.Objects.requireNonNull; import static org.elasticsearch.test.ESTestCase.awaitBusy; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.junit.Assert.assertTrue; @@ -83,6 +90,7 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { new ActionHandler<>(TestPersistentAction.INSTANCE, TransportTestPersistentAction.class), new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), + new ActionHandler<>(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class), new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), new ActionHandler<>(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class) ); @@ -109,7 +117,8 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(PersistentActionCoordinator.Status.class, PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), new NamedWriteableRegistry.Entry(ClusterState.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), - new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom) + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom), + new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) ); } @@ -243,6 +252,63 @@ public TestPersistentTaskRequestBuilder newRequestBuilder(ElasticsearchClient cl } } + public static class Status implements Task.Status { + public static final String NAME = "test"; + + private final String phase; + + public Status(String phase) { + this.phase = requireNonNull(phase, "Phase cannot be null"); + } + + public Status(StreamInput in) throws IOException { + phase = in.readString(); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("phase", phase); + builder.endObject(); + return builder; + } + + @Override + public boolean isFragment() { + return false; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(phase); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + // Implements equals and hashcode for testing + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != Status.class) { + return false; + } + Status other = (Status) obj; + return phase.equals(other.phase); + } + + @Override + public int hashCode() { + return phase.hashCode(); + } + } + public static class TransportTestPersistentAction extends TransportPersistentAction { @@ -274,14 +340,41 @@ protected void nodeOperation(PersistentTask task, TestRequest request, ActionLis logger.info("started node operation for the task {}", task); try { TestTask testTask = (TestTask) task; - assertTrue(awaitBusy(() -> testTask.isCancelled() || - testTask.getOperation() != null || - transportService.lifecycleState() != Lifecycle.State.STARTED)); // speedup finishing on closed nodes - if (transportService.lifecycleState() == Lifecycle.State.STARTED) { + AtomicInteger phase = new AtomicInteger(); + while (true) { + // wait for something to happen + assertTrue(awaitBusy(() -> testTask.isCancelled() || + testTask.getOperation() != null || + transportService.lifecycleState() != Lifecycle.State.STARTED)); // speedup finishing on closed nodes + if (transportService.lifecycleState() != Lifecycle.State.STARTED) { + return; + } if ("finish".equals(testTask.getOperation())) { listener.onResponse(Empty.INSTANCE); + return; } else if ("fail".equals(testTask.getOperation())) { listener.onFailure(new RuntimeException("Simulating failure")); + return; + } else if ("update_status".equals(testTask.getOperation())) { + testTask.setOperation(null); + CountDownLatch latch = new CountDownLatch(1); + Status status = new Status("phase " + phase.incrementAndGet()); + logger.info("updating the task status to {}", status); + updatePersistentTaskStatus(task, status, new ActionListener() { + @Override + public void onResponse(Empty empty) { + logger.info("updating was successful"); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.info("updating failed", e); + latch.countDown(); + fail(e.toString()); + } + }); + assertTrue(latch.await(10, TimeUnit.SECONDS)); } else if (testTask.isCancelled()) { // Cancellation make cause different ways for the task to finish if (randomBoolean()) { @@ -293,6 +386,7 @@ protected void nodeOperation(PersistentTask task, TestRequest request, ActionLis } else { listener.onFailure(new RuntimeException(testTask.getReasonCancelled())); } + return; } else { fail("We really shouldn't be here"); } @@ -432,8 +526,8 @@ public static class TransportTestTaskAction extends TransportTasksAction Date: Fri, 3 Feb 2017 16:07:46 +0000 Subject: [PATCH 04/50] Fix check style error after upgrade Original commit: elastic/x-pack@3bf4025f786e535754ccf84b6544b3c3415b6d73 --- .../persistent/PersistentActionCoordinator.java | 10 +++++----- .../persistent/PersistentActionCoordinatorTests.java | 4 ++-- .../persistent/TestPersistentActionPlugin.java | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java index 80faf8083bc41..c6491a36d3777 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java @@ -206,7 +206,7 @@ private void startCompletionNotification(RunningPersistentTask task, Exception e private class PersistentTaskListener implements ActionListener { private final RunningPersistentTask task; - public PersistentTaskListener(final RunningPersistentTask task) { + PersistentTaskListener(final RunningPersistentTask task) { this.task = task; } @@ -234,7 +234,7 @@ public void onFailure(Exception e) { private class PublishedResponseListener implements ActionListener { private final RunningPersistentTask task; - public PublishedResponseListener(final RunningPersistentTask task) { + PublishedResponseListener(final RunningPersistentTask task) { this.task = task; } @@ -269,7 +269,7 @@ private static class PersistentTaskId { private final long id; private final long allocationId; - public PersistentTaskId(long id, long allocationId) { + PersistentTaskId(long id, long allocationId) { this.id = id; this.allocationId = allocationId; } @@ -297,11 +297,11 @@ private static class RunningPersistentTask implements Provider { @Nullable private Exception failure; - public RunningPersistentTask(PersistentTask task, long id) { + RunningPersistentTask(PersistentTask task, long id) { this(task, id, State.STARTED); } - public RunningPersistentTask(PersistentTask task, long id, State state) { + RunningPersistentTask(PersistentTask task, long id, State state) { this.task = task; this.id = id; this.state = new AtomicReference<>(state); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java index 08aaf9dcbcd47..2454ac5d8907f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java @@ -342,7 +342,7 @@ private class Execution { private final PersistentActionRegistry.PersistentActionHolder holder; private final ActionListener listener; - public Execution(PersistentActionRequest request, PersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, + Execution(PersistentActionRequest request, PersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, ActionListener listener) { this.request = request; this.task = task; @@ -354,7 +354,7 @@ public Execution(PersistentActionRequest request, PersistentTask task, Persisten private class MockExecutor extends PersistentActionExecutor { private List executions = new ArrayList<>(); - public MockExecutor() { + MockExecutor() { super(null); } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java index f0a1beaa46169..6e94e36c5afce 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java @@ -436,11 +436,11 @@ public void setOperation(String operation) { static class TestTaskResponse implements Writeable { - public TestTaskResponse() { + TestTaskResponse() { } - public TestTaskResponse(StreamInput in) throws IOException { + TestTaskResponse(StreamInput in) throws IOException { in.readBoolean(); } From d340c190b20f0a32275a71aca881040d753995f0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 3 Feb 2017 15:20:44 -0500 Subject: [PATCH 05/50] Replace List with Map in PersistentTasksInProgress Store currently running persistent tasks in a map instead of a list. Original commit: elastic/x-pack@f88c9adef53db48d8bba4a375a22570a24da6b99 --- .../PersistentActionCoordinator.java | 2 +- .../PersistentTaskClusterService.java | 110 +++++++++--------- .../persistent/PersistentTasksInProgress.java | 84 +++++++------ .../PersistentActionCoordinatorTests.java | 47 ++++---- .../persistent/PersistentActionIT.java | 22 ++-- .../PersistentTasksInProgressTests.java | 8 +- 6 files changed, 145 insertions(+), 128 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java index c6491a36d3777..b2c0b66f7bed1 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java @@ -82,7 +82,7 @@ public void clusterChanged(ClusterChangedEvent event) { String localNodeId = event.state().getNodes().getLocalNodeId(); Set notVisitedTasks = new HashSet<>(runningTasks.keySet()); if (tasks != null) { - for (PersistentTaskInProgress taskInProgress : tasks.entries()) { + for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { if (localNodeId.equals(taskInProgress.getExecutorNode())) { PersistentTaskId persistentTaskId = new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()); RunningPersistentTask persistentTask = runningTasks.get(persistentTaskId); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java index 53c8ecefb64ec..c99b934eca57b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java @@ -30,11 +30,12 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; -import java.util.ArrayList; -import java.util.List; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -69,17 +70,13 @@ public void createPersistentTask(Strin public ClusterState execute(ClusterState currentState) throws Exception { final String executorNodeId = executorNode(action, currentState, request); PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); - final List> currentTasks = new ArrayList<>(); - final long nextId; + long nextId; if (tasksInProgress != null) { nextId = tasksInProgress.getCurrentId() + 1; - currentTasks.addAll(tasksInProgress.entries()); } else { nextId = 1; } - currentTasks.add(new PersistentTaskInProgress<>(nextId, action, request, executorNodeId)); - ClusterState.Builder builder = ClusterState.builder(currentState); - return builder.putCustom(PersistentTasksInProgress.TYPE, new PersistentTasksInProgress(nextId, currentTasks)).build(); + return createPersistentTask(currentState, new PersistentTaskInProgress<>(nextId, action, request, executorNodeId)); } @Override @@ -118,23 +115,18 @@ public ClusterState execute(ClusterState currentState) throws Exception { // Nothing to do, the task was already deleted return currentState; } - - boolean found = false; - final List> currentTasks = new ArrayList<>(); - for (PersistentTaskInProgress taskInProgress : tasksInProgress.entries()) { - if (taskInProgress.getId() == id) { - assert found == false; - found = true; - if (failure != null) { - // If the task failed - we need to restart it on another node, otherwise we just remove it - String executorNode = executorNode(taskInProgress.getAction(), currentState, taskInProgress.getRequest()); - currentTasks.add(new PersistentTaskInProgress<>(taskInProgress, executorNode)); - } - } else { - currentTasks.add(taskInProgress); + if (failure != null) { + // If the task failed - we need to restart it on another node, otherwise we just remove it + PersistentTaskInProgress taskInProgress = tasksInProgress.getTask(id); + if (taskInProgress != null) { + String executorNode = executorNode(taskInProgress.getAction(), currentState, taskInProgress.getRequest()); + return updatePersistentTask(currentState, new PersistentTaskInProgress<>(taskInProgress, executorNode)); } + return currentState; + } else { + return removePersistentTask(currentState, id); } - return rebuildClusterStateIfNeeded(found, currentState, currentTasks); + } @Override @@ -165,19 +157,11 @@ public ClusterState execute(ClusterState currentState) throws Exception { // Nothing to do, the task no longer exists return currentState; } - - boolean found = false; - final List> currentTasks = new ArrayList<>(); - for (PersistentTaskInProgress taskInProgress : tasksInProgress.entries()) { - if (taskInProgress.getId() == id) { - assert found == false; - found = true; - currentTasks.add(new PersistentTaskInProgress<>(taskInProgress, status)); - } else { - currentTasks.add(taskInProgress); - } + PersistentTaskInProgress task = tasksInProgress.getTask(id); + if (task != null) { + return updatePersistentTask(currentState, new PersistentTaskInProgress<>(task, status)); } - return rebuildClusterStateIfNeeded(found, currentState, currentTasks); + return currentState; } @Override @@ -192,14 +176,40 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } - private ClusterState rebuildClusterStateIfNeeded(boolean rebuild, ClusterState oldState, - List> currentTasks) { - if (rebuild) { + private ClusterState updatePersistentTask(ClusterState oldState, PersistentTaskInProgress newTask) { + PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); + Map> taskMap = new HashMap<>(); + taskMap.putAll(oldTasks.taskMap()); + taskMap.put(newTask.getId(), newTask); + ClusterState.Builder builder = ClusterState.builder(oldState); + PersistentTasksInProgress newTasks = new PersistentTasksInProgress(oldTasks.getCurrentId(), Collections.unmodifiableMap(taskMap)); + return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); + } + + private ClusterState createPersistentTask(ClusterState oldState, PersistentTaskInProgress newTask) { + PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); + Map> taskMap = new HashMap<>(); + if (oldTasks != null) { + taskMap.putAll(oldTasks.taskMap()); + } + taskMap.put(newTask.getId(), newTask); + ClusterState.Builder builder = ClusterState.builder(oldState); + PersistentTasksInProgress newTasks = new PersistentTasksInProgress(newTask.getId(), Collections.unmodifiableMap(taskMap)); + return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); + } + + private ClusterState removePersistentTask(ClusterState oldState, long taskId) { + PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); + if (oldTasks != null) { + Map> taskMap = new HashMap<>(); ClusterState.Builder builder = ClusterState.builder(oldState); - PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); - PersistentTasksInProgress tasks = new PersistentTasksInProgress(oldTasks.getCurrentId(), currentTasks); - return builder.putCustom(PersistentTasksInProgress.TYPE, tasks).build(); + taskMap.putAll(oldTasks.taskMap()); + taskMap.remove(taskId); + PersistentTasksInProgress newTasks = + new PersistentTasksInProgress(oldTasks.getCurrentId(), Collections.unmodifiableMap(taskMap)); + return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); } else { + // no tasks - nothing to do return oldState; } } @@ -227,7 +237,7 @@ public void clusterChanged(ClusterChangedEvent event) { // We need to check if removed nodes were running any of the tasks and reassign them boolean reassignmentRequired = false; Set removedNodes = event.nodesDelta().removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); - for (PersistentTaskInProgress taskInProgress : tasks.entries()) { + for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { if (taskInProgress.getExecutorNode() == null) { // there is an unassigned task - we need to try assigning it reassignmentRequired = true; @@ -258,22 +268,12 @@ public ClusterState execute(ClusterState currentState) throws Exception { DiscoveryNodes nodes = currentState.nodes(); if (tasks != null) { // We need to check if removed nodes were running any of the tasks and reassign them - for (PersistentTaskInProgress task : tasks.entries()) { + for (PersistentTaskInProgress task : tasks.tasks()) { if (task.getExecutorNode() == null || nodes.nodeExists(task.getExecutorNode()) == false) { // there is an unassigned task - we need to try assigning it String executorNode = executorNode(task.getAction(), currentState, task.getRequest()); if (Objects.equals(executorNode, task.getExecutorNode()) == false) { - PersistentTasksInProgress tasksInProgress = newClusterState.custom(PersistentTasksInProgress.TYPE); - final List> currentTasks = new ArrayList<>(); - for (PersistentTaskInProgress taskInProgress : tasksInProgress.entries()) { - if (task.getId() == taskInProgress.getId()) { - currentTasks.add(new PersistentTaskInProgress<>(task, executorNode)); - } else { - currentTasks.add(taskInProgress); - } - } - newClusterState = ClusterState.builder(newClusterState).putCustom(PersistentTasksInProgress.TYPE, - new PersistentTasksInProgress(tasksInProgress.getCurrentId(), currentTasks)).build(); + newClusterState = updatePersistentTask(newClusterState, new PersistentTaskInProgress<>(task, executorNode)); } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index cd29a8e484bf8..5f6a4b14e5824 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -33,7 +33,7 @@ import java.io.IOException; import java.util.Collection; -import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -44,29 +44,37 @@ public final class PersistentTasksInProgress extends AbstractNamedDiffable implements ClusterState.Custom { public static final String TYPE = "persistent_tasks"; - // TODO: Implement custom Diff for entries - private final List> entries; + // TODO: Implement custom Diff for tasks + private final Map> tasks; private final long currentId; - public PersistentTasksInProgress(long currentId, List> entries) { + public PersistentTasksInProgress(long currentId, Map> tasks) { this.currentId = currentId; - this.entries = entries; + this.tasks = tasks; } - public List> entries() { - return this.entries; + public Collection> tasks() { + return this.tasks.values(); } - public Collection> findEntries(String actionName, Predicate> predicate) { - return this.entries().stream() + public Map> taskMap() { + return this.tasks; + } + + public PersistentTaskInProgress getTask(long id) { + return this.tasks.get(id); + } + + public Collection> findTasks(String actionName, Predicate> predicate) { + return this.tasks().stream() .filter(p -> actionName.equals(p.getAction())) .filter(predicate) .collect(Collectors.toList()); } - public boolean entriesExist(String actionName, Predicate> predicate) { - return this.entries().stream() + public boolean tasksExist(String actionName, Predicate> predicate) { + return this.tasks().stream() .filter(p -> actionName.equals(p.getAction())) .anyMatch(predicate); } @@ -77,16 +85,16 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; PersistentTasksInProgress that = (PersistentTasksInProgress) o; return currentId == that.currentId && - Objects.equals(entries, that.entries); + Objects.equals(tasks, that.tasks); } @Override public int hashCode() { - return Objects.hash(entries, currentId); + return Objects.hash(tasks, currentId); } public long getNumberOfTasksOnNode(String nodeId, String action) { - return entries.stream().filter(task -> action.equals(task.action) && nodeId.equals(task.executorNode)).count(); + return tasks.values().stream().filter(task -> action.equals(task.action) && nodeId.equals(task.executorNode)).count(); } @Override @@ -97,7 +105,7 @@ public Version getMinimalSupportedVersion() { /** * A record that represents a single running persistent task */ - public static class PersistentTaskInProgress implements Writeable { + public static class PersistentTaskInProgress implements Writeable, ToXContent { private final long id; private final long allocationId; private final String action; @@ -196,6 +204,28 @@ public String getExecutorNode() { public Status getStatus() { return status; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("uuid", id); + builder.field("action", action); + builder.field("request"); + request.toXContent(builder, params); + if (status != null) { + builder.field("status", status, params); + } + builder.field("executor_node", executorNode); + } + builder.endObject(); + return builder; + } + + @Override + public boolean isFragment() { + return false; + } } @Override @@ -205,13 +235,15 @@ public String getWriteableName() { public PersistentTasksInProgress(StreamInput in) throws IOException { currentId = in.readLong(); - entries = in.readList(PersistentTaskInProgress::new); + tasks = in.readMap(StreamInput::readLong, PersistentTaskInProgress::new); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeLong(currentId); - out.writeList(entries); + out.writeMap(tasks, StreamOutput::writeLong, (stream, value) -> { + value.writeTo(stream); + }); } public static NamedDiff readDiffFrom(StreamInput in) throws IOException { @@ -227,25 +259,11 @@ public long getCurrentId() { public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.field("current_id", currentId); builder.startArray("running_tasks"); - for (PersistentTaskInProgress entry : entries) { - toXContent(entry, builder, params); + for (PersistentTaskInProgress entry : tasks.values()) { + entry.toXContent(builder, params); } builder.endArray(); return builder; } - public void toXContent(PersistentTaskInProgress entry, XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(); - { - builder.field("uuid", entry.id); - builder.field("action", entry.action); - builder.field("request"); - entry.request.toXContent(builder, params); - if (entry.status != null) { - builder.field("status", entry.status, params); - } - builder.field("executor_node", entry.executorNode); - } - builder.endObject(); - } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java index 2454ac5d8907f..6a510a55108b2 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java @@ -41,7 +41,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -85,17 +87,17 @@ public void testStartTask() throws Exception { ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); - List> tasks = new ArrayList<>(); + Map> tasks = new HashMap<>(); long taskId = randomLong(); boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { - tasks.add(new PersistentTaskInProgress<>(taskId, "test_action", new TestRequest("other_" + i), + tasks.put(taskId, new PersistentTaskInProgress<>(taskId, "test_action", new TestRequest("other_" + i), "other_node_" + randomInt(nonLocalNodesCount))); taskId++; if (added == false && randomBoolean()) { added = true; - tasks.add(new PersistentTaskInProgress<>(taskId, "test", new TestRequest("this_param"), "this_node")); + tasks.put(taskId, new PersistentTaskInProgress<>(taskId, "test", new TestRequest("this_param"), "this_node")); taskId++; } } @@ -302,38 +304,33 @@ public void sendCompletionNotification(long taskId, Exception failure, ActionLis private ClusterState addTask(ClusterState state, String action, Request request, String node) { PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); - List> tasks = prevTasks == null ? new ArrayList<>() : new ArrayList<>(prevTasks.entries()); - tasks.add(new PersistentTaskInProgress<>(prevTasks == null ? 0 : prevTasks.getCurrentId(), action, request, node)); + Map> tasks = prevTasks == null ? new HashMap<>() : new HashMap<>(prevTasks.taskMap()); + long id = prevTasks == null ? 0 : prevTasks.getCurrentId(); + tasks.put(id, new PersistentTaskInProgress<>(id, action, request, node)); return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, new PersistentTasksInProgress(prevTasks == null ? 1 : prevTasks.getCurrentId() + 1, tasks)).build(); } private ClusterState reallocateTask(ClusterState state, long taskId, String node) { PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); - List> tasks = prevTasks == null ? new ArrayList<>() : new ArrayList<>(prevTasks.entries()); - for (int i = 0; i < tasks.size(); i++) { - if (tasks.get(i).getId() == taskId) { - tasks.set(i, new PersistentTaskInProgress<>(tasks.get(i), node)); - return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, - new PersistentTasksInProgress(prevTasks == null ? 1 : prevTasks.getCurrentId() + 1, tasks)).build(); - } - } - fail("didn't find task with id " + taskId); - return null; + assertNotNull(prevTasks); + Map> tasks = new HashMap<>(prevTasks.taskMap()); + PersistentTaskInProgress prevTask = tasks.get(taskId); + assertNotNull(prevTask); + tasks.put(prevTask.getId(), new PersistentTaskInProgress<>(prevTask, node)); + return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, + new PersistentTasksInProgress(prevTasks.getCurrentId(), tasks)).build(); } private ClusterState removeTask(ClusterState state, long taskId) { PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); - List> tasks = prevTasks == null ? new ArrayList<>() : new ArrayList<>(prevTasks.entries()); - for (int i = 0; i < tasks.size(); i++) { - if (tasks.get(i).getId() == taskId) { - tasks.remove(i); - return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, - new PersistentTasksInProgress(prevTasks == null ? 1 : prevTasks.getCurrentId() + 1, tasks)).build(); - } - } - fail("didn't find task with id " + taskId); - return null; + assertNotNull(prevTasks); + Map> tasks = new HashMap<>(prevTasks.taskMap()); + PersistentTaskInProgress prevTask = tasks.get(taskId); + assertNotNull(prevTask); + tasks.remove(prevTask.getId()); + return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, + new PersistentTasksInProgress(prevTasks.getCurrentId(), tasks)).build(); } private class Execution { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java index 2c1b7ccefb9b1..4d0163321b65a 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestTasksRequestBuilder; +import org.junit.After; import java.util.Collection; import java.util.Collections; @@ -66,6 +67,11 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } + @After + public void cleanup() throws Exception { + assertNoRunningTasks(); + } + public void testPersistentActionRestart() throws Exception { long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); assertBusy(() -> { @@ -133,8 +139,6 @@ public void testPersistentActionCompletion() throws Exception { .get().getTasks().size(), equalTo(1)); } - - assertNoRunningTasks(); } public void testPersistentActionWithNoAvailableNode() throws Exception { @@ -179,8 +183,8 @@ public void testPersistentActionStatusUpdate() throws Exception { .get().getTasks().get(0); PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE); - assertThat(tasksInProgress.entries().size(), equalTo(1)); - assertThat(tasksInProgress.entries().get(0).getStatus(), nullValue()); + assertThat(tasksInProgress.tasks().size(), equalTo(1)); + assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue()); int numberOfUpdates = randomIntBetween(1, 10); for (int i = 0; i < numberOfUpdates; i++) { @@ -192,9 +196,9 @@ public void testPersistentActionStatusUpdate() throws Exception { int finalI = i; assertBusy(() -> { PersistentTasksInProgress tasks = internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE); - assertThat(tasks.entries().size(), equalTo(1)); - assertThat(tasks.entries().get(0).getStatus(), notNullValue()); - assertThat(tasks.entries().get(0).getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); + assertThat(tasks.tasks().size(), equalTo(1)); + assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue()); + assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); }); } @@ -203,8 +207,6 @@ public void testPersistentActionStatusUpdate() throws Exception { // Complete the running task and make sure it finishes properly assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) .get().getTasks().size(), equalTo(1)); - - assertNoRunningTasks(); } private void assertNoRunningTasks() throws Exception { @@ -217,7 +219,7 @@ private void assertNoRunningTasks() throws Exception { // Make sure the task is removed from the cluster state assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE)) - .entries(), empty()); + .tasks(), empty()); }); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java index e9f91740507b6..582eac2d8845e 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java @@ -27,16 +27,16 @@ import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; +import java.util.HashMap; +import java.util.Map; public class PersistentTasksInProgressTests extends AbstractWireSerializingTestCase { @Override protected PersistentTasksInProgress createTestInstance() { int numberOfTasks = randomInt(10); - List> entries = new ArrayList<>(); + Map> entries = new HashMap<>(); for (int i = 0; i < numberOfTasks; i++) { PersistentTaskInProgress taskInProgress = new PersistentTaskInProgress<>( randomLong(), randomAsciiOfLength(10), new TestPersistentActionPlugin.TestRequest(randomAsciiOfLength(10)), @@ -45,7 +45,7 @@ protected PersistentTasksInProgress createTestInstance() { // From time to time update status taskInProgress = new PersistentTaskInProgress<>(taskInProgress, new Status(randomAsciiOfLength(10))); } - entries.add(taskInProgress); + entries.put(taskInProgress.getId(), taskInProgress); } return new PersistentTasksInProgress(randomLong(), entries); } From 243b7e4499a37d0a67333fc09e7adfc3ab2bb559 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 8 Feb 2017 21:31:06 +0100 Subject: [PATCH 06/50] Moved job lifecycle over to persistent tasks Also replaced the DELETING status from JobState with a boolean flag on Job. The state of a job is now stored inside a persistent task in cluster state. Jobs that aren't running don't have a persistent task, so I moved that notion of being deleted to the job config itself. Original commit: elastic/x-pack@21cd19ca1c037158412f4582bf8a05d74ec773d4 --- .../PersistentActionCoordinator.java | 9 +++++++-- .../persistent/PersistentActionService.java | 20 +++++++++++++++++-- .../persistent/PersistentTask.java | 10 +++++++++- .../persistent/PersistentTasksInProgress.java | 2 +- .../PersistentActionCoordinatorTests.java | 11 +++++----- .../TestPersistentActionPlugin.java | 6 ++---- 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java index b2c0b66f7bed1..0825f246c3690 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; @@ -32,11 +33,11 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import java.io.IOException; import java.util.HashMap; @@ -224,7 +225,11 @@ public void onFailure(Exception e) { "cancelled task {} failed with an exception, cancellation reason [{}]", task.getId(), task.getTask().getReasonCancelled()), e); } - startCompletionNotification(task, null); + if (CancelTasksRequest.DEFAULT_REASON.equals(task.getTask().getReasonCancelled())) { + startCompletionNotification(task, null); + } else { + startCompletionNotification(task, e); + } } else { startCompletionNotification(task, e); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java index 073dfb059aab4..f8a063640fdff 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java @@ -26,8 +26,10 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; /** * Service responsible for executing restartable actions that can survive disappearance of a coordinating and executor nodes. @@ -35,11 +37,13 @@ public class PersistentActionService extends AbstractComponent { private final Client client; + private final ThreadPool threadPool; private final ClusterService clusterService; - public PersistentActionService(Settings settings, ClusterService clusterService, Client client) { + public PersistentActionService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client) { super(settings); this.client = client; + this.threadPool = threadPool; this.clusterService = clusterService; } @@ -56,8 +60,20 @@ public void sendRequest(String action, public void sendCompletionNotification(long taskId, Exception failure, ActionListener listener) { CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); + // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier. + // reason [the applied cluster state is not yet available]) try { - client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener); + threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @Override + protected void doRun() throws Exception { + client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener); + } + }); } catch (Exception e) { listener.onFailure(e); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java index 559f970ba925b..c87bbda19e991 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.inject.Provider; import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; /** @@ -40,6 +39,15 @@ public boolean shouldCancelChildrenOnCancellation() { return true; } + // In case of persistent tasks we always need to return: `false` + // because in case of persistent task the parent task isn't a task in the task manager, but in cluster state. + // This instructs the task manager not to try to kill this persistent task when the task manager cannot find + // a fake parent node id "cluster" in the cluster state + @Override + public final boolean cancelOnParentLeaving() { + return false; + } + @Override public Status getStatus() { Provider statusProvider = this.statusProvider; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index 5f6a4b14e5824..62c89fe793ebe 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -122,7 +122,7 @@ public PersistentTaskInProgress(long id, String action, Request request, String public PersistentTaskInProgress(PersistentTaskInProgress persistentTaskInProgress, String newExecutorNode) { this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId + 1L, - persistentTaskInProgress.action, persistentTaskInProgress.request, null, newExecutorNode); + persistentTaskInProgress.action, persistentTaskInProgress.request, persistentTaskInProgress.status, newExecutorNode); } public PersistentTaskInProgress(PersistentTaskInProgress persistentTaskInProgress, Status status) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java index 6a510a55108b2..253f665d60263 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java @@ -30,14 +30,14 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; -import org.elasticsearch.persistent.CompletionPersistentTaskAction.Response; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.persistent.CompletionPersistentTaskAction.Response; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; import java.io.IOException; import java.util.ArrayList; @@ -166,7 +166,7 @@ public void testTaskCancellation() { ClusterService clusterService = createClusterService(); AtomicLong capturedTaskId = new AtomicLong(); AtomicReference> capturedListener = new AtomicReference<>(); - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, null, null) { + PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, null, null, null) { @Override public void sendCancellation(long taskId, ActionListener listener) { capturedTaskId.set(taskId); @@ -238,7 +238,8 @@ public void testNotificationFailure() { AtomicLong capturedTaskId = new AtomicLong(-1L); AtomicReference capturedException = new AtomicReference<>(); AtomicReference> capturedListener = new AtomicReference<>(); - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, clusterService, null) { + PersistentActionService persistentActionService = + new PersistentActionService(Settings.EMPTY, mock(ThreadPool.class), clusterService, null) { @Override public void sendCompletionNotification(long taskId, Exception failure, ActionListener listener) { capturedTaskId.set(taskId); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java index 6e94e36c5afce..c97b451ae8538 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java @@ -56,7 +56,6 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; @@ -71,7 +70,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import static java.util.Objects.requireNonNull; import static org.elasticsearch.test.ESTestCase.awaitBusy; @@ -101,7 +99,7 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry) { - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, clusterService, client); + PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService, client); PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY); return Arrays.asList( persistentActionService, @@ -114,7 +112,7 @@ public Collection createComponents(Client client, ClusterService cluster public List getNamedWriteables() { return Arrays.asList( new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), - new NamedWriteableRegistry.Entry(PersistentActionCoordinator.Status.class, + new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), new NamedWriteableRegistry.Entry(ClusterState.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom), From 16e661c34ba85d539fd69cdf6933173a1c19331c Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 10 Feb 2017 19:58:13 -0500 Subject: [PATCH 07/50] Make persistent task persist full cluster restart This commit moves persistent tasks from ClusterState.Custom to MetaData.Custom and adds ability for the task to remain in the metadata after completion. --- .../CreatePersistentTaskAction.java | 254 +++++++++++ .../PersistentActionCoordinator.java | 17 +- .../persistent/PersistentActionService.java | 4 +- .../PersistentTaskClusterService.java | 274 +++++++----- .../persistent/PersistentTasksInProgress.java | 390 ++++++++++++++++- .../RemovePersistentTaskAction.java | 2 +- .../persistent/StartPersistentTaskAction.java | 100 +++-- .../persistent/TransportPersistentAction.java | 2 +- ...ersistentActionCoordinatorStatusTests.java | 39 ++ .../PersistentActionCoordinatorTests.java | 55 +-- .../PersistentActionFullRestartIT.java | 136 ++++++ .../persistent/PersistentActionIT.java | 106 +++-- .../PersistentTaskClusterServiceTests.java | 401 ++++++++++++++++++ .../PersistentTasksInProgressTests.java | 241 ++++++++++- .../StartPersistentActionRequestTests.java | 2 +- .../TestPersistentActionPlugin.java | 55 ++- .../UpdatePersistentTaskRequestTests.java | 47 ++ 17 files changed, 1882 insertions(+), 243 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorStatusTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java create mode 100644 server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java diff --git a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java new file mode 100644 index 0000000000000..d6a56ae928ce4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java @@ -0,0 +1,254 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * This action can be used to add the record for the persistent action to the cluster state. + */ +public class CreatePersistentTaskAction extends Action { + + public static final CreatePersistentTaskAction INSTANCE = new CreatePersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/create"; + + private CreatePersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + public static class Request extends MasterNodeRequest { + + private String action; + + private PersistentActionRequest request; + + private boolean stopped; + + private boolean removeOnCompletion = true; + + public Request() { + + } + + public Request(String action, PersistentActionRequest request) { + this.action = action; + this.request = request; + this.stopped = false; + this.removeOnCompletion = true; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + action = in.readString(); + request = in.readNamedWriteable(PersistentActionRequest.class); + stopped = in.readBoolean(); + removeOnCompletion = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(action); + out.writeNamedWriteable(request); + out.writeBoolean(stopped); + out.writeBoolean(removeOnCompletion); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (this.action == null) { + validationException = addValidationError("action must be specified", validationException); + } + if (this.request == null) { + validationException = addValidationError("request must be specified", validationException); + } + return validationException; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request1 = (Request) o; + return Objects.equals(action, request1.action) && + Objects.equals(request, request1.request) && + removeOnCompletion == request1.removeOnCompletion && + stopped == request1.stopped; + } + + @Override + public int hashCode() { + return Objects.hash(action, request, removeOnCompletion, stopped); + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public PersistentActionRequest getRequest() { + return request; + } + + public void setRequest(PersistentActionRequest request) { + this.request = request; + } + + public boolean isStopped() { + return stopped; + } + + public void setStopped(boolean stopped) { + this.stopped = stopped; + } + + public boolean shouldRemoveOnCompletion() { + return removeOnCompletion; + } + + public void setRemoveOnCompletion(boolean removeOnCompletion) { + this.removeOnCompletion = removeOnCompletion; + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction action) { + super(client, action, new Request()); + } + + public RequestBuilder setAction(String action) { + request.setAction(action); + return this; + } + + public RequestBuilder setRequest(PersistentActionRequest persistentActionRequest) { + request.setRequest(persistentActionRequest); + return this; + } + + /** + * Indicates if the persistent task should be created in the stopped state. Defaults to false. + */ + public RequestBuilder setStopped(boolean stopped) { + request.setStopped(stopped); + return this; + } + + /** + * Indicates if the persistent task record should be removed upon the first successful completion of the task. Defaults to true. + */ + public RequestBuilder setRemoveOnCompletion(boolean removeOnCompletion) { + request.setRemoveOnCompletion(removeOnCompletion); + return this; + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + PersistentActionRegistry persistentActionRegistry, + PersistentActionService persistentActionService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, CreatePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + PersistentActionExecutor executor = new PersistentActionExecutor(threadPool); + clusterService.addListener(new PersistentActionCoordinator(settings, persistentActionService, persistentActionRegistry, + transportService.getTaskManager(), executor)); + } + + @Override + protected String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + protected PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTaskClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion, + new ActionListener() { + @Override + public void onResponse(Long newTaskId) { + listener.onResponse(new PersistentActionResponse(newTaskId)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java index 0825f246c3690..ccf3a9dc35d1d 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java @@ -75,8 +75,8 @@ public PersistentActionCoordinator(Settings settings, @Override public void clusterChanged(ClusterChangedEvent event) { - PersistentTasksInProgress tasks = event.state().custom(PersistentTasksInProgress.TYPE); - PersistentTasksInProgress previousTasks = event.previousState().custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress tasks = event.state().getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress previousTasks = event.previousState().getMetaData().custom(PersistentTasksInProgress.TYPE); if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { // We have some changes let's check if they are related to our node @@ -402,6 +402,19 @@ public State getState() { public boolean isFragment() { return false; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return state == status.state; + } + + @Override + public int hashCode() { + return Objects.hash(state); + } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java index f8a063640fdff..4cd553b84c7a5 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java @@ -49,9 +49,9 @@ public PersistentActionService(Settings settings, ThreadPool threadPool, Cluster public void sendRequest(String action, Request request, ActionListener listener) { - StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(action, request); + CreatePersistentTaskAction.Request startRequest = new CreatePersistentTaskAction.Request(action, request); try { - client.execute(StartPersistentTaskAction.INSTANCE, startRequest, listener); + client.execute(CreatePersistentTaskAction.INSTANCE, startRequest, listener); } catch (Exception e) { listener.onFailure(e); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java index c99b934eca57b..a9a75863a1bd3 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java @@ -19,11 +19,14 @@ package org.elasticsearch.persistent; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; @@ -33,9 +36,6 @@ import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -63,20 +63,19 @@ public PersistentTaskClusterService(Settings settings, PersistentActionRegistry * @param request request * @param listener the listener that will be called when task is started */ - public void createPersistentTask(String action, Request request, + public void createPersistentTask(String action, Request request, boolean stopped, + boolean removeOnCompletion, ActionListener listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - final String executorNodeId = executorNode(action, currentState, request); - PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); - long nextId; - if (tasksInProgress != null) { - nextId = tasksInProgress.getCurrentId() + 1; + final String executorNodeId; + if (stopped) { + executorNodeId = null; // the task is stopped no need to assign it anywhere yet } else { - nextId = 1; + executorNodeId = executorNode(action, currentState, request); } - return createPersistentTask(currentState, new PersistentTaskInProgress<>(nextId, action, request, executorNodeId)); + return update(currentState, builder(currentState).addTask(action, request, stopped, removeOnCompletion, executorNodeId)); } @Override @@ -86,7 +85,8 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(((PersistentTasksInProgress) newState.custom(PersistentTasksInProgress.TYPE)).getCurrentId()); + listener.onResponse( + ((PersistentTasksInProgress) newState.getMetaData().custom(PersistentTasksInProgress.TYPE)).getCurrentId()); } }); } @@ -110,23 +110,81 @@ public void completeOrRestartPersistentTask(long id, Exception failure, ActionLi clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); - if (tasksInProgress == null) { - // Nothing to do, the task was already deleted - return currentState; - } - if (failure != null) { - // If the task failed - we need to restart it on another node, otherwise we just remove it - PersistentTaskInProgress taskInProgress = tasksInProgress.getTask(id); - if (taskInProgress != null) { - String executorNode = executorNode(taskInProgress.getAction(), currentState, taskInProgress.getRequest()); - return updatePersistentTask(currentState, new PersistentTaskInProgress<>(taskInProgress, executorNode)); + PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + if (tasksInProgress.hasTask(id)) { + if (failure != null) { + // If the task failed - we need to restart it on another node, otherwise we just remove it + tasksInProgress.reassignTask(id, (action, request) -> executorNode(action, currentState, request)); + } else { + tasksInProgress.finishTask(id); } + return update(currentState, tasksInProgress); + } else { + // we don't send the error message back to the caller becase that would cause an infinite loop of notifications + logger.warn("The task {} wasn't found, status is not updated", id); return currentState; + } + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(Empty.INSTANCE); + } + }); + } + + /** + * Switches the persistent task from stopped to started mode + * + * @param id the id of a persistent task + * @param listener the listener that will be called when task is removed + */ + public void startPersistentTask(long id, ActionListener listener) { + clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + if (tasksInProgress.hasTask(id)) { + return update(currentState, tasksInProgress + .assignTask(id, (action, request) -> executorNode(action, currentState, request))); } else { - return removePersistentTask(currentState, id); + throw new ResourceNotFoundException("the task with id {} doesn't exist", id); } + } + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(Empty.INSTANCE); + } + }); + } + + /** + * Removes the persistent task + * + * @param id the id of a persistent task + * @param listener the listener that will be called when task is removed + */ + public void removePersistentTask(long id, ActionListener listener) { + clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + if (tasksInProgress.hasTask(id)) { + return update(currentState, tasksInProgress.removeTask(id)); + } else { + throw new ResourceNotFoundException("the task with id {} doesn't exist", id); + } } @Override @@ -152,16 +210,12 @@ public void updatePersistentTaskStatus(long id, Task.Status status, ActionListen clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); - if (tasksInProgress == null) { - // Nothing to do, the task no longer exists - return currentState; - } - PersistentTaskInProgress task = tasksInProgress.getTask(id); - if (task != null) { - return updatePersistentTask(currentState, new PersistentTaskInProgress<>(task, status)); + PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + if (tasksInProgress.hasTask(id)) { + return update(currentState, tasksInProgress.updateTaskStatus(id, status)); + } else { + throw new ResourceNotFoundException("the task with id {} doesn't exist", id); } - return currentState; } @Override @@ -176,44 +230,6 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } - private ClusterState updatePersistentTask(ClusterState oldState, PersistentTaskInProgress newTask) { - PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); - Map> taskMap = new HashMap<>(); - taskMap.putAll(oldTasks.taskMap()); - taskMap.put(newTask.getId(), newTask); - ClusterState.Builder builder = ClusterState.builder(oldState); - PersistentTasksInProgress newTasks = new PersistentTasksInProgress(oldTasks.getCurrentId(), Collections.unmodifiableMap(taskMap)); - return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); - } - - private ClusterState createPersistentTask(ClusterState oldState, PersistentTaskInProgress newTask) { - PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); - Map> taskMap = new HashMap<>(); - if (oldTasks != null) { - taskMap.putAll(oldTasks.taskMap()); - } - taskMap.put(newTask.getId(), newTask); - ClusterState.Builder builder = ClusterState.builder(oldState); - PersistentTasksInProgress newTasks = new PersistentTasksInProgress(newTask.getId(), Collections.unmodifiableMap(taskMap)); - return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); - } - - private ClusterState removePersistentTask(ClusterState oldState, long taskId) { - PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); - if (oldTasks != null) { - Map> taskMap = new HashMap<>(); - ClusterState.Builder builder = ClusterState.builder(oldState); - taskMap.putAll(oldTasks.taskMap()); - taskMap.remove(taskId); - PersistentTasksInProgress newTasks = - new PersistentTasksInProgress(oldTasks.getCurrentId(), Collections.unmodifiableMap(taskMap)); - return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); - } else { - // no tasks - nothing to do - return oldState; - } - } - private String executorNode(String action, ClusterState currentState, Request request) { TransportPersistentAction persistentAction = registry.getPersistentActionSafe(action); persistentAction.validate(request, currentState); @@ -232,28 +248,46 @@ private String executorNode(String act @Override public void clusterChanged(ClusterChangedEvent event) { if (event.localNodeMaster()) { - PersistentTasksInProgress tasks = event.state().custom(PersistentTasksInProgress.TYPE); - if (tasks != null && (event.nodesChanged() || event.previousState().nodes().isLocalNodeElectedMaster() == false)) { - // We need to check if removed nodes were running any of the tasks and reassign them - boolean reassignmentRequired = false; - Set removedNodes = event.nodesDelta().removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); - for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { - if (taskInProgress.getExecutorNode() == null) { - // there is an unassigned task - we need to try assigning it - reassignmentRequired = true; - break; - } - if (removedNodes.contains(taskInProgress.getExecutorNode())) { - // The caller node disappeared, we need to assign a new caller node - reassignmentRequired = true; - break; + logger.trace("checking task reassignment for cluster state {}", event.state().getVersion()); + if (reassignmentRequired(event, this::executorNode)) { + logger.trace("task reassignment is needed"); + reassignTasks(); + } else { + logger.trace("task reassignment is not needed"); + } + } + } + + interface ExecutorNodeDecider { + String executorNode(String action, ClusterState currentState, Request request); + } + + static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { + PersistentTasksInProgress tasks = event.state().getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress prevTasks = event.previousState().getMetaData().custom(PersistentTasksInProgress.TYPE); + if (tasks != null && (Objects.equals(tasks, prevTasks) == false || + event.nodesChanged() || + event.routingTableChanged() || + event.previousState().nodes().isLocalNodeElectedMaster() == false)) { + // We need to check if removed nodes were running any of the tasks and reassign them + boolean reassignmentRequired = false; + Set removedNodes = event.nodesDelta().removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); + for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { + if (taskInProgress.isStopped() == false) { // skipping stopped tasks + if (taskInProgress.getExecutorNode() == null || removedNodes.contains(taskInProgress.getExecutorNode())) { + // there is an unassigned task or task with a disappeared node - we need to try assigning it + if (Objects.equals(taskInProgress.getRequest(), + decider.executorNode(taskInProgress.getAction(), event.state(), taskInProgress.getRequest())) == false) { + // it looks like a assignment for at least one task is possible - let's trigger reassignment + reassignmentRequired = true; + break; + } } } - if (reassignmentRequired) { - reassignTasks(); - } } + return reassignmentRequired; } + return false; } /** @@ -263,22 +297,7 @@ public void reassignTasks() { clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksInProgress tasks = currentState.custom(PersistentTasksInProgress.TYPE); - ClusterState newClusterState = currentState; - DiscoveryNodes nodes = currentState.nodes(); - if (tasks != null) { - // We need to check if removed nodes were running any of the tasks and reassign them - for (PersistentTaskInProgress task : tasks.tasks()) { - if (task.getExecutorNode() == null || nodes.nodeExists(task.getExecutorNode()) == false) { - // there is an unassigned task - we need to try assigning it - String executorNode = executorNode(task.getAction(), currentState, task.getRequest()); - if (Objects.equals(executorNode, task.getExecutorNode()) == false) { - newClusterState = updatePersistentTask(newClusterState, new PersistentTaskInProgress<>(task, executorNode)); - } - } - } - } - return newClusterState; + return reassignTasks(currentState, logger, PersistentTaskClusterService.this::executorNode); } @Override @@ -292,4 +311,49 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } }); } + + static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) { + PersistentTasksInProgress tasks = currentState.getMetaData().custom(PersistentTasksInProgress.TYPE); + ClusterState clusterState = currentState; + DiscoveryNodes nodes = currentState.nodes(); + if (tasks != null) { + logger.trace("reassigning {} persistent tasks", tasks.tasks().size()); + // We need to check if removed nodes were running any of the tasks and reassign them + for (PersistentTaskInProgress task : tasks.tasks()) { + if (task.isStopped() == false && + (task.getExecutorNode() == null || nodes.nodeExists(task.getExecutorNode()) == false)) { + // there is an unassigned task - we need to try assigning it + String executorNode = decider.executorNode(task.getAction(), clusterState, task.getRequest()); + if (Objects.equals(executorNode, task.getExecutorNode()) == false) { + logger.trace("reassigning task {} from node {} to node {}", task.getId(), + task.getExecutorNode(), executorNode); + clusterState = update(clusterState, builder(clusterState).reassignTask(task.getId(), executorNode)); + } else { + logger.trace("ignoring task {} because executor nodes are the same {}", task.getId(), executorNode); + } + } else { + if (task.isStopped()) { + logger.trace("ignoring task {} because it is stopped", task.getId()); + } else { + logger.trace("ignoring task {} because it is still running", task.getId()); + } + } + } + } + return clusterState; + } + + private static PersistentTasksInProgress.Builder builder(ClusterState currentState) { + return PersistentTasksInProgress.builder(currentState.getMetaData().custom(PersistentTasksInProgress.TYPE)); + } + + private static ClusterState update(ClusterState currentState, PersistentTasksInProgress.Builder tasksInProgress) { + if (tasksInProgress.isChanged()) { + return ClusterState.builder(currentState).metaData( + MetaData.builder(currentState.metaData()).putCustom(PersistentTasksInProgress.TYPE, tasksInProgress.build()) + ).build(); + } else { + return currentState; + } + } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index 62c89fe793ebe..13e1a6f07e67d 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -20,30 +20,44 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractNamedDiffable; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.Task.Status; import java.io.IOException; import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.MetaData.ALL_CONTEXTS; + /** * A cluster state record that contains a list of all running persistent tasks */ -public final class PersistentTasksInProgress extends AbstractNamedDiffable implements ClusterState.Custom { +public final class PersistentTasksInProgress extends AbstractNamedDiffable implements MetaData.Custom { public static final String TYPE = "persistent_tasks"; + private static final String API_CONTEXT = MetaData.XContentContext.API.toString(); + // TODO: Implement custom Diff for tasks private final Map> tasks; @@ -54,6 +68,69 @@ public PersistentTasksInProgress(long currentId, Map PERSISTENT_TASKS_IN_PROGRESS_PARSER = new ObjectParser<>(TYPE, + Builder::new); + + public static final ObjectParser, Void> PERSISTENT_TASK_IN_PROGRESS_PARSER = + new ObjectParser<>("running_tasks", TaskBuilder::new); + + public static final NamedObjectParser, Void> ACTION_PARSER; + + static { + // Tasks parser initialization + PERSISTENT_TASKS_IN_PROGRESS_PARSER.declareLong(Builder::setCurrentId, new ParseField("current_id")); + PERSISTENT_TASKS_IN_PROGRESS_PARSER.declareObjectArray(Builder::setTasks, PERSISTENT_TASK_IN_PROGRESS_PARSER, + new ParseField("running_tasks")); + + // Action parser initialization + ObjectParser, String> parser = new ObjectParser<>("named"); + parser.declareObject(ActionDescriptionBuilder::setRequest, + (p, c) -> p.namedObject(PersistentActionRequest.class, c, null), new ParseField("request")); + parser.declareObject(ActionDescriptionBuilder::setStatus, + (p, c) -> p.namedObject(Status.class, c, null), new ParseField("status")); + ACTION_PARSER = (XContentParser p, Void c, String name) -> parser.parse(p, new ActionDescriptionBuilder<>(name), name); + + // Task parser initialization + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareLong(TaskBuilder::setId, new ParseField("id")); + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id")); + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareBoolean(TaskBuilder::setRemoveOnCompletion, new ParseField("remove_on_completion")); + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareBoolean(TaskBuilder::setStopped, new ParseField("stopped")); + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareNamedObjects( + (TaskBuilder taskBuilder, List> objects) -> { + if (objects.size() != 1) { + throw new IllegalArgumentException("only one action description per task is allowed"); + } + ActionDescriptionBuilder builder = objects.get(0); + taskBuilder.setAction(builder.action); + taskBuilder.setRequest(builder.request); + taskBuilder.setStatus(builder.status); + }, ACTION_PARSER, new ParseField("action")); + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareStringOrNull(TaskBuilder::setExecutorNode, new ParseField("executor_node")); + } + + /** + * Private builder used in XContent parser + */ + private static class ActionDescriptionBuilder { + private final String action; + private Request request; + private Status status; + + private ActionDescriptionBuilder(String action) { + this.action = action; + } + + private ActionDescriptionBuilder setRequest(Request request) { + this.request = request; + return this; + } + + private ActionDescriptionBuilder setStatus(Status status) { + this.status = status; + return this; + } + } + public Collection> tasks() { return this.tasks.values(); } @@ -93,6 +170,11 @@ public int hashCode() { return Objects.hash(tasks, currentId); } + @Override + public String toString() { + return Strings.toString(this); + } + public long getNumberOfTasksOnNode(String nodeId, String action) { return tasks.values().stream().filter(task -> action.equals(task.action) && nodeId.equals(task.executorNode)).count(); } @@ -102,6 +184,15 @@ public Version getMinimalSupportedVersion() { return Version.V_5_3_0_UNRELEASED; } + @Override + public EnumSet context() { + return ALL_CONTEXTS; + } + + public static PersistentTasksInProgress fromXContent(XContentParser parser) throws IOException { + return PERSISTENT_TASKS_IN_PROGRESS_PARSER.parse(parser, null).build(); + } + /** * A record that represents a single running persistent task */ @@ -110,32 +201,37 @@ public static class PersistentTaskInProgress persistentTaskInProgress, String newExecutorNode) { - this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId + 1L, - persistentTaskInProgress.action, persistentTaskInProgress.request, persistentTaskInProgress.status, newExecutorNode); + public PersistentTaskInProgress(PersistentTaskInProgress task, boolean stopped, String newExecutorNode) { + this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status, + newExecutorNode); } - public PersistentTaskInProgress(PersistentTaskInProgress persistentTaskInProgress, Status status) { - this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId, - persistentTaskInProgress.action, persistentTaskInProgress.request, status, persistentTaskInProgress.executorNode); + public PersistentTaskInProgress(PersistentTaskInProgress task, Status status) { + this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, task.executorNode); } - private PersistentTaskInProgress(long id, long allocationId, String action, Request request, Status status, String executorNode) { + private PersistentTaskInProgress(long id, long allocationId, String action, Request request, + boolean stopped, boolean removeOnCompletion, Status status, String executorNode) { this.id = id; this.allocationId = allocationId; this.action = action; this.request = request; this.status = status; + this.stopped = stopped; + this.removeOnCompletion = removeOnCompletion; this.executorNode = executorNode; // Update parent request for starting tasks with correct parent task ID request.setParentTask("cluster", id); @@ -147,6 +243,8 @@ private PersistentTaskInProgress(StreamInput in) throws IOException { allocationId = in.readLong(); action = in.readString(); request = (Request) in.readNamedWriteable(PersistentActionRequest.class); + stopped = in.readBoolean(); + removeOnCompletion = in.readBoolean(); status = in.readOptionalNamedWriteable(Task.Status.class); executorNode = in.readOptionalString(); } @@ -157,6 +255,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(allocationId); out.writeString(action); out.writeNamedWriteable(request); + out.writeBoolean(stopped); + out.writeBoolean(removeOnCompletion); out.writeOptionalNamedWriteable(status); out.writeOptionalString(executorNode); } @@ -170,13 +270,20 @@ public boolean equals(Object o) { allocationId == that.allocationId && Objects.equals(action, that.action) && Objects.equals(request, that.request) && + stopped == that.stopped && + removeOnCompletion == that.removeOnCompletion && Objects.equals(status, that.status) && Objects.equals(executorNode, that.executorNode); } @Override public int hashCode() { - return Objects.hash(id, allocationId, action, request, status, executorNode); + return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, executorNode); + } + + @Override + public String toString() { + return Strings.toString(this); } public long getId() { @@ -205,18 +312,39 @@ public Status getStatus() { return status; } + public boolean isStopped() { + return stopped; + } + + public boolean shouldRemoveOnCompletion() { + return removeOnCompletion; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - builder.field("uuid", id); - builder.field("action", action); - builder.field("request"); - request.toXContent(builder, params); - if (status != null) { - builder.field("status", status, params); + builder.field("id", id); + builder.startObject("action"); + { + builder.startObject(action); + { + builder.field("request"); + request.toXContent(builder, params); + if (status != null) { + builder.field("status", status, params); + } + } + builder.endObject(); + } + builder.endObject(); + if (API_CONTEXT.equals(params.param(MetaData.CONTEXT_MODE_PARAM, API_CONTEXT))) { + // These are transient values that shouldn't be persisted to gateway cluster state or snapshot + builder.field("allocation_id", allocationId); + builder.field("executor_node", executorNode); } - builder.field("executor_node", executorNode); + builder.field("stopped", stopped); + builder.field("remove_on_completion", removeOnCompletion); } builder.endObject(); return builder; @@ -228,6 +356,62 @@ public boolean isFragment() { } } + private static class TaskBuilder { + private long id; + private long allocationId; + private String action; + private Request request; + private boolean stopped = true; + private boolean removeOnCompletion; + private Status status; + private String executorNode; + + public TaskBuilder setId(long id) { + this.id = id; + return this; + } + + public TaskBuilder setAllocationId(long allocationId) { + this.allocationId = allocationId; + return this; + } + + public TaskBuilder setAction(String action) { + this.action = action; + return this; + } + + public TaskBuilder setRequest(Request request) { + this.request = request; + return this; + } + + public TaskBuilder setStatus(Status status) { + this.status = status; + return this; + } + + + public TaskBuilder setStopped(boolean stopped) { + this.stopped = stopped; + return this; + } + + public TaskBuilder setRemoveOnCompletion(boolean removeOnCompletion) { + this.removeOnCompletion = removeOnCompletion; + return this; + } + + public TaskBuilder setExecutorNode(String executorNode) { + this.executorNode = executorNode; + return this; + } + + public PersistentTaskInProgress build() { + return new PersistentTaskInProgress<>(id, allocationId, action, request, stopped, removeOnCompletion, status, executorNode); + } + } + @Override public String getWriteableName() { return TYPE; @@ -246,8 +430,8 @@ public void writeTo(StreamOutput out) throws IOException { }); } - public static NamedDiff readDiffFrom(StreamInput in) throws IOException { - return readDiffFrom(ClusterState.Custom.class, TYPE, in); + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(MetaData.Custom.class, TYPE, in); } public long getCurrentId() { @@ -266,4 +450,168 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par return builder; } + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(PersistentTasksInProgress tasks) { + return new Builder(tasks); + } + + public static class Builder { + private final Map> tasks = new HashMap<>(); + private long currentId; + private boolean changed; + + public Builder() { + } + + public Builder(PersistentTasksInProgress tasksInProgress) { + if (tasksInProgress != null) { + tasks.putAll(tasksInProgress.tasks); + currentId = tasksInProgress.currentId; + } else { + currentId = 0; + } + } + + private Builder setCurrentId(long currentId) { + this.currentId = currentId; + return this; + } + + private Builder setTasks(List> tasks) { + for (TaskBuilder builder : tasks) { + PersistentTaskInProgress task = builder.build(); + this.tasks.put(task.getId(), task); + } + return this; + } + + /** + * Adds a new task to the builder + *

+ * After the task is added its id can be found by calling {{@link #getCurrentId()}} method. + */ + public Builder addTask(String action, Request request, boolean stopped, + boolean removeOnCompletion, String executorNode) { + changed = true; + currentId++; + tasks.put(currentId, new PersistentTaskInProgress<>(currentId, action, request, stopped, removeOnCompletion, + executorNode)); + return this; + } + + /** + * Reassigns the task to another node if the task exist + */ + public Builder reassignTask(long taskId, String executorNode) { + PersistentTaskInProgress taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + changed = true; + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, executorNode)); + } + return this; + } + + /** + * Assigns the task to another node if the task exist and not currently assigned + *

+ * The operation is only performed if the task is not currently assigned to any nodes. To force assignment use + * {@link #reassignTask(long, BiFunction)} instead + */ + @SuppressWarnings("unchecked") + public Builder assignTask(long taskId, + BiFunction executorNodeFunc) { + PersistentTaskInProgress taskInProgress = (PersistentTaskInProgress) tasks.get(taskId); + if (taskInProgress != null && taskInProgress.getExecutorNode() == null) { // only assign unassigned tasks + String executorNode = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + if (executorNode != null || taskInProgress.isStopped()) { + changed = true; + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, executorNode)); + } + } + return this; + } + + /** + * Reassigns the task to another node if the task exist + */ + @SuppressWarnings("unchecked") + public Builder reassignTask(long taskId, + BiFunction executorNodeFunc) { + PersistentTaskInProgress taskInProgress = (PersistentTaskInProgress) tasks.get(taskId); + if (taskInProgress != null) { + changed = true; + String executorNode = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, executorNode)); + } + return this; + } + + /** + * Updates the task status if the task exist + */ + public Builder updateTaskStatus(long taskId, Status status) { + PersistentTaskInProgress taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + changed = true; + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, status)); + } + return this; + } + + /** + * Removes the task if the task exist + */ + public Builder removeTask(long taskId) { + if (tasks.remove(taskId) != null) { + changed = true; + } + return this; + } + + /** + * Finishes the task if the task exist. + * + * If the task is marked with removeOnCompletion flag, it is removed from the list, otherwise it is stopped. + */ + public Builder finishTask(long taskId) { + PersistentTaskInProgress taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + changed = true; + if (taskInProgress.removeOnCompletion) { + tasks.remove(taskId); + } else { + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, true, null)); + } + } + return this; + } + + /** + * Checks if the task is currently present in the list + */ + public boolean hasTask(long taskId) { + return tasks.containsKey(taskId); + } + + /** + * Returns the id of the last added task + */ + public long getCurrentId() { + return currentId; + } + + /** + * Returns true if any the task list was changed since the builder was created + */ + public boolean isChanged() { + return changed; + } + + public PersistentTasksInProgress build() { + return new PersistentTasksInProgress(currentId, Collections.unmodifiableMap(tasks)); + } + } } diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java index 3cbd1f585279d..08dbd8a5e7afb 100644 --- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -191,7 +191,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, null, new ActionListener() { + persistentTaskClusterService.removePersistentTask(request.taskId, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 1ee89b097ba6c..8c302ab170e6e 100644 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -36,16 +37,17 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.Objects; /** - * Internal action used by TransportPersistentAction to add the record for the persistent action to the cluster state. + * This action can be used to start persistent action previously created using {@link CreatePersistentTaskAction} */ public class StartPersistentTaskAction extends Action { public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction(); @@ -61,37 +63,36 @@ public RequestBuilder newRequestBuilder(ElasticsearchClient client) { } @Override - public PersistentActionResponse newResponse() { - return new PersistentActionResponse(); + public Response newResponse() { + return new Response(); } public static class Request extends MasterNodeRequest { - private String action; - - private PersistentActionRequest request; + private long taskId; public Request() { } - public Request(String action, PersistentActionRequest request) { - this.action = action; - this.request = request; + public Request(long taskId) { + this.taskId = taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - action = in.readString(); - request = in.readOptionalNamedWriteable(PersistentActionRequest.class); + taskId = in.readLong(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(action); - out.writeOptionalNamedWriteable(request); + out.writeLong(taskId); } @Override @@ -103,26 +104,65 @@ public ActionRequestValidationException validate() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Request request1 = (Request) o; - return Objects.equals(action, request1.action) && - Objects.equals(request, request1.request); + Request request = (Request) o; + return taskId == request.taskId; + } + + @Override + public int hashCode() { + return Objects.hash(taskId); + } + } + + public static class Response extends AcknowledgedResponse { + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeAcknowledged(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AcknowledgedResponse that = (AcknowledgedResponse) o; + return isAcknowledged() == that.isAcknowledged(); } @Override public int hashCode() { - return Objects.hash(action, request); + return Objects.hash(isAcknowledged()); } + } public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + StartPersistentTaskAction.Response, StartPersistentTaskAction.RequestBuilder> { protected RequestBuilder(ElasticsearchClient client, StartPersistentTaskAction action) { super(client, action, new Request()); } + + public final RequestBuilder setTaskId(long taskId) { + request.setTaskId(taskId); + return this; + } + } - public static class TransportAction extends TransportMasterNodeAction { + public static class TransportAction extends TransportMasterNodeAction { private final PersistentTaskClusterService persistentTaskClusterService; @@ -130,25 +170,20 @@ public static class TransportAction extends TransportMasterNodeAction listener) { - persistentTaskClusterService.createPersistentTask(request.action, request.request, new ActionListener() { + protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + persistentTaskClusterService.startPersistentTask(request.taskId, new ActionListener() { @Override - public void onResponse(Long newTaskId) { - listener.onResponse(new PersistentActionResponse(newTaskId)); + public void onResponse(Empty empty) { + listener.onResponse(new Response(true)); } @Override diff --git a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java index 7e98262a6d6e2..9e58ee0896df1 100644 --- a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java @@ -71,7 +71,7 @@ public DiscoveryNode executorNode(Request request, ClusterState clusterState) { protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate selector) { long minLoad = Long.MAX_VALUE; DiscoveryNode minLoadedNode = null; - PersistentTasksInProgress persistentTasksInProgress = clusterState.custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress persistentTasksInProgress = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); for (DiscoveryNode node : clusterState.getNodes()) { if (selector.test(node)) { if (persistentTasksInProgress == null) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorStatusTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorStatusTests.java new file mode 100644 index 0000000000000..fbb649d17fce5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorStatusTests.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.persistent.PersistentActionCoordinator.State; +import org.elasticsearch.persistent.PersistentActionCoordinator.Status; + +import static org.hamcrest.Matchers.containsString; + +public class PersistentActionCoordinatorStatusTests extends AbstractWireSerializingTestCase { + + @Override + protected Status createTestInstance() { + return new Status(randomFrom(State.values())); + } + + @Override + protected Writeable.Reader instanceReader() { + return Status::new; + } + + public void testToString() { + assertThat(createTestInstance().toString(), containsString("state")); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java index 253f665d60263..7de67a22caefd 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; @@ -36,14 +37,11 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.CompletionPersistentTaskAction.Response; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -87,18 +85,14 @@ public void testStartTask() throws Exception { ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); - Map> tasks = new HashMap<>(); - long taskId = randomLong(); + PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder(); boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { - tasks.put(taskId, new PersistentTaskInProgress<>(taskId, "test_action", new TestRequest("other_" + i), - "other_node_" + randomInt(nonLocalNodesCount))); - taskId++; + tasks.addTask("test_action", new TestRequest("other_" + i), false, true, "other_node_" + randomInt(nonLocalNodesCount)); if (added == false && randomBoolean()) { added = true; - tasks.put(taskId, new PersistentTaskInProgress<>(taskId, "test", new TestRequest("this_param"), "this_node")); - taskId++; + tasks.addTask("test", new TestRequest("this_param"), false, true, "this_node"); } } } @@ -107,8 +101,9 @@ public void testStartTask() throws Exception { logger.info("No local node action was added"); } - ClusterState newClusterState = ClusterState.builder(state) - .putCustom(PersistentTasksInProgress.TYPE, new PersistentTasksInProgress(taskId, tasks)).build(); + MetaData.Builder metaData = MetaData.builder(state.metaData()); + metaData.putCustom(PersistentTasksInProgress.TYPE, tasks.build()); + ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); if (added) { @@ -304,34 +299,26 @@ public void sendCompletionNotification(long taskId, Exception failure, ActionLis private ClusterState addTask(ClusterState state, String action, Request request, String node) { - PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); - Map> tasks = prevTasks == null ? new HashMap<>() : new HashMap<>(prevTasks.taskMap()); - long id = prevTasks == null ? 0 : prevTasks.getCurrentId(); - tasks.put(id, new PersistentTaskInProgress<>(id, action, request, node)); - return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, - new PersistentTasksInProgress(prevTasks == null ? 1 : prevTasks.getCurrentId() + 1, tasks)).build(); + PersistentTasksInProgress.Builder builder = + PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, + builder.addTask(action, request, false, true, node).build())).build(); } private ClusterState reallocateTask(ClusterState state, long taskId, String node) { - PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); - assertNotNull(prevTasks); - Map> tasks = new HashMap<>(prevTasks.taskMap()); - PersistentTaskInProgress prevTask = tasks.get(taskId); - assertNotNull(prevTask); - tasks.put(prevTask.getId(), new PersistentTaskInProgress<>(prevTask, node)); - return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, - new PersistentTasksInProgress(prevTasks.getCurrentId(), tasks)).build(); + PersistentTasksInProgress.Builder builder = + PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); + assertTrue(builder.hasTask(taskId)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, + builder.reassignTask(taskId, node).build())).build(); } private ClusterState removeTask(ClusterState state, long taskId) { - PersistentTasksInProgress prevTasks = state.custom(PersistentTasksInProgress.TYPE); - assertNotNull(prevTasks); - Map> tasks = new HashMap<>(prevTasks.taskMap()); - PersistentTaskInProgress prevTask = tasks.get(taskId); - assertNotNull(prevTask); - tasks.remove(prevTask.getId()); - return ClusterState.builder(state).putCustom(PersistentTasksInProgress.TYPE, - new PersistentTasksInProgress(prevTasks.getCurrentId(), tasks)).build(); + PersistentTasksInProgress.Builder builder = + PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); + assertTrue(builder.hasTask(taskId)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, + builder.removeTask(taskId).build())).build(); } private class Execution { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java new file mode 100644 index 0000000000000..6c9e7e7e3e7f0 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; + +import java.util.Collection; +import java.util.Collections; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1) +public class PersistentActionFullRestartIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(TestPersistentActionPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + protected boolean ignoreExternalCluster() { + return true; + } + + @TestLogging("org.elasticsearch.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG") + public void testFullClusterRestart() throws Exception { + int numberOfTasks = randomIntBetween(1, 10); + long[] taskIds = new long[numberOfTasks]; + boolean[] stopped = new boolean[numberOfTasks]; + int runningTasks = 0; + for (int i = 0; i < numberOfTasks; i++) { + if (randomBoolean()) { + runningTasks++; + taskIds[i] = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); + stopped[i] = false; + } else { + taskIds[i] = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client()) + .setAction(TestPersistentAction.NAME) + .setRequest(new TestRequest("Blah")) + .setStopped(true) + .get().getTaskId(); + stopped[i] = true; + } + } + final int numberOfRunningTasks = runningTasks; + PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksInProgress.TYPE); + assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); + + if (numberOfRunningTasks > 0) { + // Make sure that at least one of the tasks is running + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + .getTasks().size(), greaterThan(0)); + }); + } + + // Restart cluster + internalCluster().fullRestart(); + ensureYellow(); + + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksInProgress.TYPE); + assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); + // Check that cluster state is correct + for (int i = 0; i < numberOfTasks; i++) { + PersistentTaskInProgress task = tasksInProgress.getTask(taskIds[i]); + assertNotNull(task); + assertThat(task.isStopped(), equalTo(stopped[i])); + } + + logger.info("Waiting for {} original tasks to start", numberOfRunningTasks); + assertBusy(() -> { + // Wait for the running task to start automatically + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), + equalTo(numberOfRunningTasks)); + }); + + // Start all other tasks + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksInProgress.TYPE); + for (int i = 0; i < numberOfTasks; i++) { + PersistentTaskInProgress task = tasksInProgress.getTask(taskIds[i]); + assertNotNull(task); + logger.info("checking task with id {} stopped {} node {}", task.getId(), task.isStopped(), task.getExecutorNode()); + assertThat(task.isStopped(), equalTo(stopped[i])); + assertThat(task.getExecutorNode(), stopped[i] ? nullValue() : notNullValue()); + if (stopped[i]) { + assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(task.getId()).get()); + } + } + + logger.info("Waiting for {} tasks to start", numberOfTasks); + assertBusy(() -> { + // Wait for all tasks to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), + equalTo(numberOfTasks)); + }); + + logger.info("Complete all tasks"); + // Complete the running task and make sure it finishes properly + assertThat(new TestPersistentActionPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(), + equalTo(numberOfTasks)); + + assertBusy(() -> { + // Make sure the task is removed from the cluster state + assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksInProgress.TYPE)).tasks(), empty()); + }); + + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java index 4d0163321b65a..ec288c15a1c1b 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; @@ -51,22 +52,10 @@ protected Collection> transportClientPlugins() { return nodePlugins(); } - @Override - protected Collection> getMockPlugins() { - return super.getMockPlugins(); - } - protected boolean ignoreExternalCluster() { return true; } - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .build(); - } - @After public void cleanup() throws Exception { assertNoRunningTasks(); @@ -126,19 +115,64 @@ public void testPersistentActionCompletion() throws Exception { // Verifying parent assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(taskId)); assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); + stopOrCancelTask(firstRunningTask.getTaskId()); + } - if (randomBoolean()) { - logger.info("Completing the running task"); - // Complete the running task and make sure it finishes properly - assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) - .get().getTasks().size(), equalTo(1)); - } else { - logger.info("Cancelling the running task"); - // Cancel the running task and make sure it finishes properly - assertThat(client().admin().cluster().prepareCancelTasks().setTaskId(firstRunningTask.getTaskId()) - .get().getTasks().size(), equalTo(1)); + public void testPersistentActionCompletionWithoutRemoval() throws Exception { + boolean stopped = randomBoolean(); + long taskId = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client()) + .setAction(TestPersistentAction.NAME) + .setRequest(new TestPersistentActionPlugin.TestRequest("Blah")) + .setRemoveOnCompletion(false) + .setStopped(stopped).get().getTaskId(); + + PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksInProgress.TYPE); + assertThat(tasksInProgress.tasks().size(), equalTo(1)); + assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped)); + assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue()); + assertThat(tasksInProgress.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); + + int numberOfIters = randomIntBetween(1, 5); // we will start/stop the action a few times before removing it + logger.info("start/stop the task {} times stating with stopped {}", numberOfIters, stopped); + for (int i = 0; i < numberOfIters; i++) { + logger.info("iteration {}", i); + if (stopped) { + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), + empty()); + assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + } + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks() + .size(), equalTo(1)); + }); + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + .get().getTasks().get(0); + stopOrCancelTask(firstRunningTask.getTaskId()); + + assertBusy(() -> { + // Wait for the task to finish + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + .getTasks(); + logger.info("Found {} tasks", tasks.size()); + assertThat(tasks.size(), equalTo(0)); + }); + stopped = true; } + + assertBusy(() -> { + // Wait for the task to be marked as stopped + PersistentTasksInProgress tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksInProgress.TYPE); + assertThat(tasks.tasks().size(), equalTo(1)); + assertThat(tasks.getTask(taskId).isStopped(), equalTo(true)); + assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); + }); + + logger.info("Removing action record from cluster state"); + assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); } public void testPersistentActionWithNoAvailableNode() throws Exception { @@ -182,7 +216,8 @@ public void testPersistentActionStatusUpdate() throws Exception { TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") .get().getTasks().get(0); - PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksInProgress.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue()); @@ -195,7 +230,8 @@ public void testPersistentActionStatusUpdate() throws Exception { int finalI = i; assertBusy(() -> { - PersistentTasksInProgress tasks = internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksInProgress.TYPE); assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue()); assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); @@ -209,6 +245,24 @@ public void testPersistentActionStatusUpdate() throws Exception { .get().getTasks().size(), equalTo(1)); } + + private void stopOrCancelTask(TaskId taskId) { + if (randomBoolean()) { + logger.info("Completing the running task"); + // Complete the running task and make sure it finishes properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(taskId) + .get().getTasks().size(), equalTo(1)); + + } else { + logger.info("Cancelling the running task"); + // Cancel the running task and make sure it finishes properly + assertThat(client().admin().cluster().prepareCancelTasks().setTaskId(taskId) + .get().getTasks().size(), equalTo(1)); + } + + + } + private void assertNoRunningTasks() throws Exception { assertBusy(() -> { // Wait for the task to finish @@ -218,8 +272,8 @@ private void assertNoRunningTasks() throws Exception { assertThat(tasks.size(), equalTo(0)); // Make sure the task is removed from the cluster state - assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().custom(PersistentTasksInProgress.TYPE)) - .tasks(), empty()); + assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksInProgress.TYPE)).tasks(), empty()); }); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java new file mode 100644 index 0000000000000..fb6e884a1bfc2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java @@ -0,0 +1,401 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + */ + +package org.elasticsearch.persistent; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class PersistentTaskClusterServiceTests extends ESTestCase { + + public void testReassignmentRequired() { + int numberOfIterations = randomIntBetween(10, 100); + ClusterState clusterState = initialState(); + for (int i = 0; i < numberOfIterations; i++) { + boolean significant = randomBoolean(); + ClusterState previousState = clusterState; + logger.info("inter {} significant: {}", i, significant); + if (significant) { + clusterState = significantChange(clusterState); + } else { + clusterState = insignificantChange(clusterState); + } + ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState); + assertThat(dumpEvent(event), significant, equalTo(PersistentTaskClusterService.reassignmentRequired(event, + new PersistentTaskClusterService.ExecutorNodeDecider() { + @Override + public String executorNode( + String action, ClusterState currentState, Request request) { + return randomNode(currentState.nodes()); + } + }))); + } + } + + public void testReassignTasksWithNoTasks() { + ClusterState clusterState = initialState(); + assertThat(reassign(clusterState).metaData().custom(PersistentTasksInProgress.TYPE), nullValue()); + } + + public void testReassignConsidersClusterStateUpdates() { + ClusterState clusterState = initialState(); + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder( + clusterState.metaData().custom(PersistentTasksInProgress.TYPE)); + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); + addTestNodes(nodes, randomIntBetween(1, 10)); + int numberOfTasks = randomIntBetween(2, 40); + for (int i = 0; i < numberOfTasks; i++) { + addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false); + } + + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksInProgress.TYPE, tasks.build()); + clusterState = builder.metaData(metaData).nodes(nodes).build(); + ClusterState newClusterState = reassign(clusterState); + + PersistentTasksInProgress tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + assertThat(tasksInProgress, notNullValue()); + + } + + public void testReassignTasks() { + ClusterState clusterState = initialState(); + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder( + clusterState.metaData().custom(PersistentTasksInProgress.TYPE)); + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); + addTestNodes(nodes, randomIntBetween(1, 10)); + int numberOfTasks = randomIntBetween(0, 40); + for (int i = 0; i < numberOfTasks; i++) { + switch (randomInt(3)) { + case 0: + // add an unassigned task that should get assigned because it's assigned to a non-existing node or unassigned + addTask(tasks, "should_assign", "assign_me", randomBoolean() ? null : "no_longer_exits", false); + break; + case 1: + // add a task assigned to non-existing node that should not get assigned + addTask(tasks, "should_not_assign", "dont_assign_me", randomBoolean() ? null : "no_longer_exits", false); + break; + case 2: + // add a stopped task assigned to non-existing node that should not get assigned + addTask(tasks, "should_not_assign", "fail_me_if_called", null, true); + break; + case 3: + addTask(tasks, "assign_one", "assign_one", randomBoolean() ? null : "no_longer_exits", false); + break; + + } + } + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksInProgress.TYPE, tasks.build()); + clusterState = builder.metaData(metaData).nodes(nodes).build(); + ClusterState newClusterState = reassign(clusterState); + + PersistentTasksInProgress tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + assertThat(tasksInProgress, notNullValue()); + + assertThat("number of tasks shouldn't change as a result or reassignment", + numberOfTasks, equalTo(tasksInProgress.tasks().size())); + + int assignOneCount = 0; + + for (PersistentTaskInProgress task : tasksInProgress.tasks()) { + if (task.isStopped()) { + assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue()); + } else { + switch (task.getAction()) { + case "should_assign": + assertThat(task.getExecutorNode(), notNullValue()); + if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { + logger.info(clusterState.metaData().custom(PersistentTasksInProgress.TYPE).toString()); + } + assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), + clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); + break; + case "should_not_assign": + assertThat(task.getExecutorNode(), nullValue()); + break; + case "assign_one": + if (task.getExecutorNode() != null) { + assignOneCount++; + assertThat("more than one assign_one tasks are assigned", assignOneCount, lessThanOrEqualTo(1)); + } + break; + default: + fail("Unknown action " + task.getAction()); + } + } + } + } + + + private void addTestNodes(DiscoveryNodes.Builder nodes, int nonLocalNodesCount) { + for (int i = 0; i < nonLocalNodesCount; i++) { + nodes.add(new DiscoveryNode("other_node_" + i, buildNewFakeTransportAddress(), Version.CURRENT)); + } + } + + private ClusterState reassign(ClusterState clusterState) { + return PersistentTaskClusterService.reassignTasks(clusterState, logger, + new PersistentTaskClusterService.ExecutorNodeDecider() { + @Override + public String executorNode( + String action, ClusterState currentState, Request request) { + TestRequest testRequest = (TestRequest) request; + switch (testRequest.getTestParam()) { + case "assign_me": + return randomNode(currentState.nodes()); + case "dont_assign_me": + return null; + case "fail_me_if_called": + fail("the decision decider shouldn't be called on this task"); + return null; + case "assign_one": + return assignOnlyOneTaskAtATime(currentState); + default: + fail("unknown param " + testRequest.getTestParam()); + } + return null; + } + }); + + } + + private String assignOnlyOneTaskAtATime(ClusterState clusterState) { + DiscoveryNodes nodes = clusterState.nodes(); + PersistentTasksInProgress tasksInProgress = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + if (tasksInProgress.findTasks("assign_one", + task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) { + return randomNode(clusterState.nodes()); + } else { + return null; + } + } + + private String randomNode(DiscoveryNodes nodes) { + if (nodes.getNodes().isEmpty()) { + return null; + } + List nodeList = new ArrayList<>(); + for (ObjectCursor node : nodes.getNodes().keys()) { + nodeList.add(node.value); + } + return randomFrom(nodeList); + + } + + private String dumpEvent(ClusterChangedEvent event) { + return "nodes_changed: " + event.nodesChanged() + + " nodes_removed:" + event.nodesRemoved() + + " routing_table_changed:" + event.routingTableChanged() + + " tasks: " + event.state().metaData().custom(PersistentTasksInProgress.TYPE); + } + + private ClusterState significantChange(ClusterState clusterState) { + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksInProgress tasks = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + if (tasks != null) { + if (randomBoolean()) { + // + boolean removedNode = false; + for (PersistentTaskInProgress task : tasks.tasks()) { + if (task.getExecutorNode() != null && clusterState.nodes().nodeExists(task.getExecutorNode())) { + logger.info("removed node {}", task.getExecutorNode()); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(task.getExecutorNode())); + return builder.build(); + } + } + } + } + boolean tasksOrNodesChanged = false; + // add a new unassigned task + if (hasUnassigned(tasks, clusterState.nodes()) == false) { + // we don't have any unassigned tasks - add some + logger.info("added random task"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), null, false); + tasksOrNodesChanged = true; + } + // add a node if there are unassigned tasks + if (clusterState.nodes().getNodes().isEmpty()) { + logger.info("added random node"); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAsciiOfLength(10)))); + tasksOrNodesChanged = true; + } + + if (tasksOrNodesChanged == false) { + // change routing table to simulate a change + logger.info("changed routing table"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); + RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); + changeRoutingTable(metaData, routingTable); + builder.metaData(metaData).routingTable(routingTable.build()); + } + return builder.build(); + } + + private ClusterState insignificantChange(ClusterState clusterState) { + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksInProgress tasks = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + if (randomBoolean()) { + if (hasUnassigned(tasks, clusterState.nodes()) == false) { + // we don't have any unassigned tasks - adding a node or changing a routing table shouldn't affect anything + if (randomBoolean()) { + logger.info("added random node"); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAsciiOfLength(10)))); + } + if (randomBoolean()) { + // add unassigned task in stopped state + logger.info("added random stopped task"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), null, true); + return builder.build(); + } else { + logger.info("changed routing table"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); + RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); + changeRoutingTable(metaData, routingTable); + builder.metaData(metaData).routingTable(routingTable.build()); + } + return builder.build(); + } + } + if (randomBoolean()) { + // remove a node that doesn't have any tasks assigned to it and it's not the master node + for (DiscoveryNode node : clusterState.nodes()) { + if (hasTasksAssignedTo(tasks, node.getId()) == false && "this_node".equals(node.getId()) == false) { + logger.info("removed unassigned node {}", node.getId()); + return builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(node.getId())).build(); + } + } + } + + if (randomBoolean()) { + // clear the task + if (randomBoolean()) { + logger.info("removed all tasks"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksInProgress.TYPE, + PersistentTasksInProgress.builder().build()); + return builder.metaData(metaData).build(); + } else { + logger.info("set task custom to null"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasksInProgress.TYPE); + return builder.metaData(metaData).build(); + } + } + logger.info("removed all unassigned tasks and changed routing table"); + PersistentTasksInProgress.Builder tasksBuilder = PersistentTasksInProgress.builder(tasks); + if (tasks != null) { + for (PersistentTaskInProgress task : tasks.tasks()) { + if (task.getExecutorNode() == null) { + tasksBuilder.removeTask(task.getId()); + } + } + } + // Just add a random index - that shouldn't change anything + IndexMetaData indexMetaData = IndexMetaData.builder(randomAsciiOfLength(10)) + .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false) + .putCustom(PersistentTasksInProgress.TYPE, tasksBuilder.build()); + return builder.metaData(metaData).build(); + } + + private boolean hasUnassigned(PersistentTasksInProgress tasks, DiscoveryNodes discoveryNodes) { + if (tasks == null || tasks.tasks().isEmpty()) { + return false; + } + return tasks.tasks().stream().anyMatch(task -> + task.isStopped() == false && + (task.getExecutorNode() == null || discoveryNodes.nodeExists(task.getExecutorNode()))); + } + + private boolean hasTasksAssignedTo(PersistentTasksInProgress tasks, String nodeId) { + return tasks != null && tasks.tasks().stream().anyMatch( + task -> nodeId.equals(task.getExecutorNode())) == false; + } + + private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, + MetaData.Builder metaData, PersistentTasksInProgress.Builder tasks, + String node, + boolean stopped) { + return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksInProgress.TYPE, + tasks.addTask(randomAsciiOfLength(10), new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), node).build())); + } + + private void addTask(PersistentTasksInProgress.Builder tasks, String action, String param, String node, boolean stopped) { + tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), node); + } + + private DiscoveryNode newNode(String nodeId) { + return new DiscoveryNode(nodeId, buildNewFakeTransportAddress(), emptyMap(), + Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), + Version.CURRENT); + } + + + private ClusterState initialState() { + MetaData.Builder metaData = MetaData.builder(); + RoutingTable.Builder routingTable = RoutingTable.builder(); + int randomIndices = randomIntBetween(0, 5); + for (int i = 0; i < randomIndices; i++) { + changeRoutingTable(metaData, routingTable); + } + + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); + nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), "this_node")); + nodes.localNodeId("this_node"); + nodes.masterNodeId("this_node"); + + return ClusterState.builder(ClusterName.DEFAULT) + .metaData(metaData) + .routingTable(routingTable.build()) + .build(); + } + + private void changeRoutingTable(MetaData.Builder metaData, RoutingTable.Builder routingTable) { + IndexMetaData indexMetaData = IndexMetaData.builder(randomAsciiOfLength(10)) + .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + metaData.put(indexMetaData, false); + routingTable.addAsNew(indexMetaData); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java index 582eac2d8845e..896d749772a54 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java @@ -18,48 +18,263 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaData.Custom; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.AbstractDiffableSerializationTestCase; +import org.elasticsearch.persistent.PersistentTasksInProgress.Builder; import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; +import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.Collections; -public class PersistentTasksInProgressTests extends AbstractWireSerializingTestCase { +import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; +import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; + +public class PersistentTasksInProgressTests extends AbstractDiffableSerializationTestCase { @Override protected PersistentTasksInProgress createTestInstance() { int numberOfTasks = randomInt(10); - Map> entries = new HashMap<>(); + PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder(); for (int i = 0; i < numberOfTasks; i++) { - PersistentTaskInProgress taskInProgress = new PersistentTaskInProgress<>( - randomLong(), randomAsciiOfLength(10), new TestPersistentActionPlugin.TestRequest(randomAsciiOfLength(10)), - randomAsciiOfLength(10)); + tasks.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), + randomBoolean(), randomBoolean(), randomAsciiOfLength(10)); if (randomBoolean()) { // From time to time update status - taskInProgress = new PersistentTaskInProgress<>(taskInProgress, new Status(randomAsciiOfLength(10))); + tasks.updateTaskStatus(tasks.getCurrentId(), new Status(randomAsciiOfLength(10))); } - entries.put(taskInProgress.getId(), taskInProgress); } - return new PersistentTasksInProgress(randomLong(), entries); + return tasks.build(); } @Override - protected Writeable.Reader instanceReader() { + protected Writeable.Reader instanceReader() { return PersistentTasksInProgress::new; } @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Arrays.asList( - new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestPersistentActionPlugin.TestRequest::new), + new Entry(MetaData.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), + new Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom), + new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), new Entry(Task.Status.class, Status.NAME, Status::new) )); } + + @Override + protected Custom makeTestChanges(Custom testInstance) { + PersistentTasksInProgress tasksInProgress = (PersistentTasksInProgress) testInstance; + Builder builder = new Builder(); + switch (randomInt(3)) { + case 0: + addRandomTask(builder); + break; + case 1: + if (tasksInProgress.tasks().isEmpty()) { + addRandomTask(builder); + } else { + builder.reassignTask(pickRandomTask(tasksInProgress), randomAsciiOfLength(10)); + } + break; + case 2: + if (tasksInProgress.tasks().isEmpty()) { + addRandomTask(builder); + } else { + builder.updateTaskStatus(pickRandomTask(tasksInProgress), randomBoolean() ? new Status(randomAsciiOfLength(10)) : null); + } + break; + case 3: + if (tasksInProgress.tasks().isEmpty()) { + addRandomTask(builder); + } else { + builder.removeTask(pickRandomTask(tasksInProgress)); + } + break; + } + return builder.build(); + } + + @Override + protected Writeable.Reader> diffReader() { + return PersistentTasksInProgress::readDiffFrom; + } + + @Override + protected PersistentTasksInProgress doParseInstance(XContentParser parser) throws IOException { + return PersistentTasksInProgress.fromXContent(parser); + } + + @Override + protected XContentBuilder toXContent(Custom instance, XContentType contentType) throws IOException { + return toXContent(instance, contentType, new ToXContent.MapParams( + Collections.singletonMap(MetaData.CONTEXT_MODE_PARAM, MetaData.XContentContext.API.toString()))); + } + + protected XContentBuilder toXContent(Custom instance, XContentType contentType, ToXContent.MapParams params) throws IOException { + // We need all attribute to be serialized/de-serialized for testing + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (randomBoolean()) { + builder.prettyPrint(); + } + if (instance.isFragment()) { + builder.startObject(); + } + instance.toXContent(builder, params); + if (instance.isFragment()) { + builder.endObject(); + } + return builder; + } + + private Builder addRandomTask(Builder builder) { + builder.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), + randomBoolean(), randomBoolean(), randomAsciiOfLength(10)); + return builder; + } + + private long pickRandomTask(PersistentTasksInProgress testInstance) { + return randomFrom(new ArrayList<>(testInstance.tasks())).getId(); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return new NamedXContentRegistry(Arrays.asList( + new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), + TestRequest::fromXContent), + new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(TestPersistentAction.NAME), + Status::fromXContent) + )); + } + + @SuppressWarnings("unchecked") + public void testSerializationContext() throws Exception { + PersistentTasksInProgress testInstance = createTestInstance(); + for (int i = 0; i < randomInt(10); i++) { + testInstance = (PersistentTasksInProgress) makeTestChanges(testInstance); + } + + ToXContent.MapParams params = new ToXContent.MapParams( + Collections.singletonMap(MetaData.CONTEXT_MODE_PARAM, randomFrom(CONTEXT_MODE_SNAPSHOT, CONTEXT_MODE_GATEWAY))); + + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = toXContent(testInstance, xContentType, params); + XContentBuilder shuffled = shuffleXContent(builder); + + XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes()); + PersistentTasksInProgress newInstance = doParseInstance(parser); + assertNotSame(newInstance, testInstance); + + assertEquals(testInstance.tasks().size(), newInstance.tasks().size()); + for (PersistentTaskInProgress testTask : testInstance.tasks()) { + PersistentTaskInProgress newTask = (PersistentTaskInProgress) newInstance.getTask(testTask.getId()); + assertNotNull(newTask); + + // Things that should be serialized + assertEquals(testTask.getAction(), newTask.getAction()); + assertEquals(testTask.getId(), newTask.getId()); + assertEquals(testTask.getStatus(), newTask.getStatus()); + assertEquals(testTask.getRequest(), newTask.getRequest()); + assertEquals(testTask.isStopped(), newTask.isStopped()); + + // Things that shouldn't be serialized + assertEquals(0, newTask.getAllocationId()); + assertNull(newTask.getExecutorNode()); + } + } + + public void testBuilder() { + PersistentTasksInProgress persistentTasksInProgress = null; + long lastKnownTask = -1; + for (int i = 0; i < randomIntBetween(10, 100); i++) { + final Builder builder; + if (randomBoolean()) { + builder = new Builder(); + } else { + builder = new Builder(persistentTasksInProgress); + } + boolean changed = false; + for (int j = 0; j < randomIntBetween(1, 10); j++) { + switch (randomInt(5)) { + case 0: + lastKnownTask = addRandomTask(builder).getCurrentId(); + changed = true; + break; + case 1: + if (builder.hasTask(lastKnownTask)) { + changed = true; + } + if (randomBoolean()) { + builder.reassignTask(lastKnownTask, randomAsciiOfLength(10)); + } else { + builder.reassignTask(lastKnownTask, (s, request) -> randomAsciiOfLength(10)); + } + break; + case 2: + if (builder.hasTask(lastKnownTask)) { + PersistentTaskInProgress task = builder.build().getTask(lastKnownTask); + if (randomBoolean()) { + // Trying to reassign to the same node + builder.assignTask(lastKnownTask, (s, request) -> task.getExecutorNode()); + // should change if the task was stopped AND unassigned + if (task.getExecutorNode() == null && task.isStopped()) { + changed = true; + } + } else { + // Trying to reassign to a different node + builder.assignTask(lastKnownTask, (s, request) -> randomAsciiOfLength(10)); + // should change if the task was unassigned + if (task.getExecutorNode() == null) { + changed = true; + } + } + } else { + // task doesn't exist - shouldn't change + builder.assignTask(lastKnownTask, (s, request) -> randomAsciiOfLength(10)); + } + break; + case 3: + if (builder.hasTask(lastKnownTask)) { + changed = true; + } + builder.updateTaskStatus(lastKnownTask, randomBoolean() ? new Status(randomAsciiOfLength(10)) : null); + break; + case 4: + if (builder.hasTask(lastKnownTask)) { + changed = true; + } + builder.removeTask(lastKnownTask); + break; + case 5: + if (builder.hasTask(lastKnownTask)) { + changed = true; + } + builder.finishTask(lastKnownTask); + break; + } + } + assertEquals(changed, builder.isChanged()); + persistentTasksInProgress = builder.build(); + } + + } + } diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java index 52be69cbeb885..17d8ca6954323 100644 --- a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.elasticsearch.persistent.StartPersistentTaskAction.Request; +import org.elasticsearch.persistent.CreatePersistentTaskAction.Request; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; import org.elasticsearch.test.AbstractStreamableTestCase; diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java index c97b451ae8538..bc8651dad24fb 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java @@ -37,8 +37,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.inject.Inject; @@ -47,8 +49,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; @@ -72,6 +76,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static java.util.Objects.requireNonNull; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.test.ESTestCase.awaitBusy; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.junit.Assert.assertTrue; @@ -87,6 +92,7 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { return Arrays.asList( new ActionHandler<>(TestPersistentAction.INSTANCE, TransportTestPersistentAction.class), new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), + new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class), new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), new ActionHandler<>(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class), new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), @@ -114,14 +120,32 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), - new NamedWriteableRegistry.Entry(ClusterState.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom), new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) ); } + @Override + public List getNamedXContent() { + return Arrays.asList( + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksInProgress.TYPE), + PersistentTasksInProgress::fromXContent), + new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), + TestRequest::fromXContent), + new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) + ); + } + public static class TestRequest extends PersistentActionRequest { + public static final ConstructingObjectParser REQUEST_PARSER = + new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new TestRequest((String) args[0])); + + static { + REQUEST_PARSER.declareString(constructorArg(), new ParseField("param")); + } + private String executorNodeAttr = null; private String responseNode = null; @@ -185,10 +209,15 @@ public void readFrom(StreamInput in) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field("param", testParam); builder.endObject(); return builder; } + public static TestRequest fromXContent(XContentParser parser) throws IOException { + return REQUEST_PARSER.parse(parser, null); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -255,6 +284,13 @@ public static class Status implements Task.Status { private final String phase; + public static final ConstructingObjectParser STATUS_PARSER = + new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new Status((String) args[0])); + + static { + STATUS_PARSER.declareString(constructorArg(), new ParseField("phase")); + } + public Status(String phase) { this.phase = requireNonNull(phase, "Phase cannot be null"); } @@ -276,6 +312,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static Task.Status fromXContent(XContentParser parser) throws IOException { + return STATUS_PARSER.parse(parser, null); + } + + @Override public boolean isFragment() { return false; @@ -319,7 +360,7 @@ public TransportTestPersistentAction(Settings settings, ThreadPool threadPool, T IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, TestPersistentAction.NAME, false, threadPool, transportService, persistentActionService, persistentActionRegistry, actionFilters, indexNameExpressionResolver, TestRequest::new, - ThreadPool.Names.MANAGEMENT); + ThreadPool.Names.GENERIC); this.transportService = transportService; } @@ -342,8 +383,9 @@ protected void nodeOperation(PersistentTask task, TestRequest request, ActionLis while (true) { // wait for something to happen assertTrue(awaitBusy(() -> testTask.isCancelled() || - testTask.getOperation() != null || - transportService.lifecycleState() != Lifecycle.State.STARTED)); // speedup finishing on closed nodes + testTask.getOperation() != null || + transportService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes + 30, TimeUnit.SECONDS)); // This can take a while during large cluster restart if (transportService.lifecycleState() != Lifecycle.State.STARTED) { return; } @@ -430,6 +472,11 @@ public String getOperation() { public void setOperation(String operation) { this.operation = operation; } + + @Override + public String toString() { + return "TestTask[" + this.getId() + ", " + this.getParentTaskId() + ", " + this.getOperation() + "]"; + } } static class TestTaskResponse implements Writeable { diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java new file mode 100644 index 0000000000000..751286eafa09d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; +import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction.Request; + +import java.util.Collections; + +public class UpdatePersistentTaskRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomLong(), new Status(randomAsciiOfLength(10))); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.singletonList( + new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) + )); + } +} From 479429c6efba51975b9792c5b8b6769788736b4b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 17 Feb 2017 14:16:33 +0100 Subject: [PATCH 08/50] In order to keep track of restarted tasks, `allocationIdOnLastStatusUpdate` field was added to `PersistentTaskInProgress` class. This will allow persistent task implementors to detect whether the executor node has changed or has been unset since the last status update has occured. --- .../persistent/PersistentTask.java | 1 + .../persistent/PersistentTasksInProgress.java | 43 ++++++++++++++++--- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java index c87bbda19e991..973b7a8836212 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java @@ -70,4 +70,5 @@ public long getPersistentTaskId() { public void setPersistentTaskId(long persistentTaskId) { this.persistentTaskId = persistentTaskId; } + } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index 13e1a6f07e67d..610d32eee4012 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -106,6 +106,8 @@ public PersistentTasksInProgress(long currentId, Map task, boolean stopped, String newExecutorNode) { this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status, - newExecutorNode); + newExecutorNode, task.allocationId); } public PersistentTaskInProgress(PersistentTaskInProgress task, Status status) { - this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, task.executorNode); + this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, + task.executorNode, task.allocationId); } private PersistentTaskInProgress(long id, long allocationId, String action, Request request, - boolean stopped, boolean removeOnCompletion, Status status, String executorNode) { + boolean stopped, boolean removeOnCompletion, Status status, + String executorNode, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; this.action = action; @@ -233,6 +239,7 @@ private PersistentTaskInProgress(long id, long allocationId, String action, Requ this.stopped = stopped; this.removeOnCompletion = removeOnCompletion; this.executorNode = executorNode; + this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; // Update parent request for starting tasks with correct parent task ID request.setParentTask("cluster", id); } @@ -247,6 +254,7 @@ private PersistentTaskInProgress(StreamInput in) throws IOException { removeOnCompletion = in.readBoolean(); status = in.readOptionalNamedWriteable(Task.Status.class); executorNode = in.readOptionalString(); + allocationIdOnLastStatusUpdate = in.readOptionalLong(); } @Override @@ -259,6 +267,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(removeOnCompletion); out.writeOptionalNamedWriteable(status); out.writeOptionalString(executorNode); + out.writeOptionalLong(allocationIdOnLastStatusUpdate); } @Override @@ -273,12 +282,14 @@ public boolean equals(Object o) { stopped == that.stopped && removeOnCompletion == that.removeOnCompletion && Objects.equals(status, that.status) && - Objects.equals(executorNode, that.executorNode); + Objects.equals(executorNode, that.executorNode) && + Objects.equals(allocationIdOnLastStatusUpdate, that.allocationIdOnLastStatusUpdate); } @Override public int hashCode() { - return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, executorNode); + return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, executorNode, + allocationIdOnLastStatusUpdate); } @Override @@ -320,6 +331,14 @@ public boolean shouldRemoveOnCompletion() { return removeOnCompletion; } + /** + * @return Whether the task status isn't stale. When a task gets unassigned from the executor node or assigned + * to a new executor node and the status hasn't been updated then the task status is stale. + */ + public boolean isCurrentStatus() { + return allocationIdOnLastStatusUpdate == allocationId; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -342,6 +361,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws // These are transient values that shouldn't be persisted to gateway cluster state or snapshot builder.field("allocation_id", allocationId); builder.field("executor_node", executorNode); + if (allocationIdOnLastStatusUpdate != null) { + builder.field("allocation_id_on_last_status_update", allocationIdOnLastStatusUpdate); + } } builder.field("stopped", stopped); builder.field("remove_on_completion", removeOnCompletion); @@ -365,6 +387,7 @@ private static class TaskBuilder { private boolean removeOnCompletion; private Status status; private String executorNode; + private Long allocationIdOnLastStatusUpdate; public TaskBuilder setId(long id) { this.id = id; @@ -407,8 +430,14 @@ public TaskBuilder setExecutorNode(String executorNode) { return this; } + public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdOnLastStatusUpdate) { + this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; + return this; + } + public PersistentTaskInProgress build() { - return new PersistentTaskInProgress<>(id, allocationId, action, request, stopped, removeOnCompletion, status, executorNode); + return new PersistentTaskInProgress<>(id, allocationId, action, request, stopped, removeOnCompletion, status, + executorNode, allocationIdOnLastStatusUpdate); } } From 5eeb480d97a9f0e93f760971263d9b39c4b3f5e7 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 21 Feb 2017 21:05:50 -0500 Subject: [PATCH 09/50] Add persistent task assignment explanations. This commit allows persistent actions to indicate why a task was or wasn't assigned to a certain node. --- .../PersistentTaskClusterService.java | 65 ++++---- .../persistent/PersistentTasksInProgress.java | 147 +++++++++++++----- .../persistent/TransportPersistentAction.java | 13 +- .../PersistentActionCoordinatorTests.java | 11 +- .../PersistentTaskClusterServiceTests.java | 137 +++++++++++----- .../PersistentTasksInProgressTests.java | 37 +++-- .../TestPersistentActionPlugin.java | 13 +- 7 files changed, 292 insertions(+), 131 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java index a9a75863a1bd3..54e8391f4d7e5 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java @@ -34,11 +34,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; /** * Component that runs only on the master node and is responsible for assigning running tasks to nodes @@ -69,13 +68,13 @@ public void createPersistentTask(Strin clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - final String executorNodeId; + final Assignment assignment; if (stopped) { - executorNodeId = null; // the task is stopped no need to assign it anywhere yet + assignment = PersistentTasksInProgress.FINISHED_TASK_ASSIGNMENT; // the task is stopped no need to assign it anywhere } else { - executorNodeId = executorNode(action, currentState, request); + assignment = getAssignement(action, currentState, request); } - return update(currentState, builder(currentState).addTask(action, request, stopped, removeOnCompletion, executorNodeId)); + return update(currentState, builder(currentState).addTask(action, request, stopped, removeOnCompletion, assignment)); } @Override @@ -114,7 +113,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (tasksInProgress.hasTask(id)) { if (failure != null) { // If the task failed - we need to restart it on another node, otherwise we just remove it - tasksInProgress.reassignTask(id, (action, request) -> executorNode(action, currentState, request)); + tasksInProgress.reassignTask(id, (action, request) -> getAssignement(action, currentState, request)); } else { tasksInProgress.finishTask(id); } @@ -151,7 +150,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress - .assignTask(id, (action, request) -> executorNode(action, currentState, request))); + .assignTask(id, (action, request) -> getAssignement(action, currentState, request))); } else { throw new ResourceNotFoundException("the task with id {} doesn't exist", id); } @@ -230,26 +229,17 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } - private String executorNode(String action, ClusterState currentState, Request request) { + private Assignment getAssignement(String action, ClusterState currentState, Request request) { TransportPersistentAction persistentAction = registry.getPersistentActionSafe(action); persistentAction.validate(request, currentState); - DiscoveryNode executorNode = persistentAction.executorNode(request, currentState); - final String executorNodeId; - if (executorNode == null) { - // The executor node not available yet, we will create task with empty executor node and try - // again later - executorNodeId = null; - } else { - executorNodeId = executorNode.getId(); - } - return executorNodeId; + return persistentAction.getAssignment(request, currentState); } @Override public void clusterChanged(ClusterChangedEvent event) { if (event.localNodeMaster()) { logger.trace("checking task reassignment for cluster state {}", event.state().getVersion()); - if (reassignmentRequired(event, this::executorNode)) { + if (reassignmentRequired(event, this::getAssignement)) { logger.trace("task reassignment is needed"); reassignTasks(); } else { @@ -259,7 +249,7 @@ public void clusterChanged(ClusterChangedEvent event) { } interface ExecutorNodeDecider { - String executorNode(String action, ClusterState currentState, Request request); + Assignment getAssignment(String action, ClusterState currentState, Request request); } static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { @@ -271,18 +261,16 @@ static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecid event.previousState().nodes().isLocalNodeElectedMaster() == false)) { // We need to check if removed nodes were running any of the tasks and reassign them boolean reassignmentRequired = false; - Set removedNodes = event.nodesDelta().removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { - if (taskInProgress.isStopped() == false) { // skipping stopped tasks - if (taskInProgress.getExecutorNode() == null || removedNodes.contains(taskInProgress.getExecutorNode())) { - // there is an unassigned task or task with a disappeared node - we need to try assigning it - if (Objects.equals(taskInProgress.getRequest(), - decider.executorNode(taskInProgress.getAction(), event.state(), taskInProgress.getRequest())) == false) { - // it looks like a assignment for at least one task is possible - let's trigger reassignment - reassignmentRequired = true; - break; - } + if (taskInProgress.needsReassignment(event.state().nodes())) { + // there is an unassigned task or task with a disappeared node - we need to try assigning it + if (Objects.equals(taskInProgress.getAssignment(), + decider.getAssignment(taskInProgress.getAction(), event.state(), taskInProgress.getRequest())) == false) { + // it looks like a assignment for at least one task is possible - let's trigger reassignment + reassignmentRequired = true; + break; } + } } return reassignmentRequired; @@ -297,7 +285,7 @@ public void reassignTasks() { clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - return reassignTasks(currentState, logger, PersistentTaskClusterService.this::executorNode); + return reassignTasks(currentState, logger, PersistentTaskClusterService.this::getAssignement); } @Override @@ -320,16 +308,15 @@ static ClusterState reassignTasks(ClusterState currentState, Logger logger, Exec logger.trace("reassigning {} persistent tasks", tasks.tasks().size()); // We need to check if removed nodes were running any of the tasks and reassign them for (PersistentTaskInProgress task : tasks.tasks()) { - if (task.isStopped() == false && - (task.getExecutorNode() == null || nodes.nodeExists(task.getExecutorNode()) == false)) { + if (task.needsReassignment(nodes)) { // there is an unassigned task - we need to try assigning it - String executorNode = decider.executorNode(task.getAction(), clusterState, task.getRequest()); - if (Objects.equals(executorNode, task.getExecutorNode()) == false) { + Assignment assignment = decider.getAssignment(task.getAction(), clusterState, task.getRequest()); + if (Objects.equals(assignment, task.getAssignment()) == false) { logger.trace("reassigning task {} from node {} to node {}", task.getId(), - task.getExecutorNode(), executorNode); - clusterState = update(clusterState, builder(clusterState).reassignTask(task.getId(), executorNode)); + task.getAssignment().getExecutorNode(), assignment.getExecutorNode()); + clusterState = update(clusterState, builder(clusterState).reassignTask(task.getId(), assignment)); } else { - logger.trace("ignoring task {} because executor nodes are the same {}", task.getId(), executorNode); + logger.trace("ignoring task {} because assignment is the same {}", task.getId(), assignment); } } else { if (task.isStopped()) { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index 610d32eee4012..9dcea474d982f 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -22,12 +22,14 @@ import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser; import org.elasticsearch.common.xcontent.ToXContent; @@ -49,6 +51,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.MetaData.ALL_CONTEXTS; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; /** * A cluster state record that contains a list of all running persistent tasks @@ -74,6 +77,9 @@ public PersistentTasksInProgress(long currentId, Map, Void> PERSISTENT_TASK_IN_PROGRESS_PARSER = new ObjectParser<>("running_tasks", TaskBuilder::new); + public static final ConstructingObjectParser ASSIGNMENT_PARSER = + new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); + public static final NamedObjectParser, Void> ACTION_PARSER; static { @@ -90,6 +96,10 @@ public PersistentTasksInProgress(long currentId, Map p.namedObject(Status.class, c, null), new ParseField("status")); ACTION_PARSER = (XContentParser p, Void c, String name) -> parser.parse(p, new ActionDescriptionBuilder<>(name), name); + // Assignment parser + ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("executor_node")); + ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("explanation")); + // Task parser initialization PERSISTENT_TASK_IN_PROGRESS_PARSER.declareLong(TaskBuilder::setId, new ParseField("id")); PERSISTENT_TASK_IN_PROGRESS_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id")); @@ -105,7 +115,7 @@ public PersistentTasksInProgress(long currentId, Map action.equals(task.action) && nodeId.equals(task.executorNode)).count(); + return tasks.values().stream().filter(task -> action.equals(task.action) && nodeId.equals(task.assignment.executorNode)).count(); } @Override @@ -195,6 +205,54 @@ public static PersistentTasksInProgress fromXContent(XContentParser parser) thro return PERSISTENT_TASKS_IN_PROGRESS_PARSER.parse(parser, null).build(); } + public static class Assignment { + @Nullable + private final String executorNode; + private final String explanation; + + public Assignment(String executorNode, String explanation) { + this.executorNode = executorNode; + assert explanation != null; + this.explanation = explanation; + } + + @Nullable + public String getExecutorNode() { + return executorNode; + } + + public String getExplanation() { + return explanation; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Assignment that = (Assignment) o; + return Objects.equals(executorNode, that.executorNode) && + Objects.equals(explanation, that.explanation); + } + + @Override + public int hashCode() { + return Objects.hash(executorNode, explanation); + } + + public boolean isAssigned() { + return executorNode != null; + } + + @Override + public String toString() { + return "node: [" + executorNode + "], explanation: [" + explanation +"]"; + } + } + + public static final Assignment INITIAL_ASSIGNMENT = new Assignment(null, "waiting for initial assignment"); + + public static final Assignment FINISHED_TASK_ASSIGNMENT = new Assignment(null, "task has finished"); + /** * A record that represents a single running persistent task */ @@ -207,30 +265,29 @@ public static class PersistentTaskInProgress task, boolean stopped, String newExecutorNode) { + public PersistentTaskInProgress(PersistentTaskInProgress task, boolean stopped, Assignment assignment) { this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status, - newExecutorNode, task.allocationId); + assignment, task.allocationId); } public PersistentTaskInProgress(PersistentTaskInProgress task, Status status) { this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, - task.executorNode, task.allocationId); + task.assignment, task.allocationId); } private PersistentTaskInProgress(long id, long allocationId, String action, Request request, boolean stopped, boolean removeOnCompletion, Status status, - String executorNode, Long allocationIdOnLastStatusUpdate) { + Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; this.action = action; @@ -238,7 +295,7 @@ private PersistentTaskInProgress(long id, long allocationId, String action, Requ this.status = status; this.stopped = stopped; this.removeOnCompletion = removeOnCompletion; - this.executorNode = executorNode; + this.assignment = assignment; this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; // Update parent request for starting tasks with correct parent task ID request.setParentTask("cluster", id); @@ -253,7 +310,7 @@ private PersistentTaskInProgress(StreamInput in) throws IOException { stopped = in.readBoolean(); removeOnCompletion = in.readBoolean(); status = in.readOptionalNamedWriteable(Task.Status.class); - executorNode = in.readOptionalString(); + assignment = new Assignment(in.readOptionalString(), in.readString()); allocationIdOnLastStatusUpdate = in.readOptionalLong(); } @@ -266,7 +323,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(stopped); out.writeBoolean(removeOnCompletion); out.writeOptionalNamedWriteable(status); - out.writeOptionalString(executorNode); + out.writeOptionalString(assignment.executorNode); + out.writeString(assignment.explanation); out.writeOptionalLong(allocationIdOnLastStatusUpdate); } @@ -282,13 +340,13 @@ public boolean equals(Object o) { stopped == that.stopped && removeOnCompletion == that.removeOnCompletion && Objects.equals(status, that.status) && - Objects.equals(executorNode, that.executorNode) && + Objects.equals(assignment, that.assignment) && Objects.equals(allocationIdOnLastStatusUpdate, that.allocationIdOnLastStatusUpdate); } @Override public int hashCode() { - return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, executorNode, + return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, assignment, allocationIdOnLastStatusUpdate); } @@ -315,7 +373,22 @@ public Request getRequest() { @Nullable public String getExecutorNode() { - return executorNode; + return assignment.executorNode; + } + + public Assignment getAssignment() { + return assignment; + } + + public boolean isAssigned() { + return assignment.isAssigned(); + } + + /** + * Returns true if the tasks is not stopped and unassigned or assigned to a non-existing node. + */ + public boolean needsReassignment(DiscoveryNodes nodes) { + return isStopped() == false && (assignment.isAssigned() == false || nodes.nodeExists(assignment.getExecutorNode()) == false); } @Nullable @@ -360,7 +433,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (API_CONTEXT.equals(params.param(MetaData.CONTEXT_MODE_PARAM, API_CONTEXT))) { // These are transient values that shouldn't be persisted to gateway cluster state or snapshot builder.field("allocation_id", allocationId); - builder.field("executor_node", executorNode); + builder.startObject("assignment"); + { + builder.field("executor_node", assignment.executorNode); + builder.field("explanation", assignment.explanation); + } + builder.endObject(); if (allocationIdOnLastStatusUpdate != null) { builder.field("allocation_id_on_last_status_update", allocationIdOnLastStatusUpdate); } @@ -386,7 +464,7 @@ private static class TaskBuilder { private boolean stopped = true; private boolean removeOnCompletion; private Status status; - private String executorNode; + private Assignment assignment = INITIAL_ASSIGNMENT; private Long allocationIdOnLastStatusUpdate; public TaskBuilder setId(long id) { @@ -425,8 +503,8 @@ public TaskBuilder setRemoveOnCompletion(boolean removeOnCompletion) { return this; } - public TaskBuilder setExecutorNode(String executorNode) { - this.executorNode = executorNode; + public TaskBuilder setAssignment(Assignment assignment) { + this.assignment = assignment; return this; } @@ -437,7 +515,7 @@ public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdO public PersistentTaskInProgress build() { return new PersistentTaskInProgress<>(id, allocationId, action, request, stopped, removeOnCompletion, status, - executorNode, allocationIdOnLastStatusUpdate); + assignment, allocationIdOnLastStatusUpdate); } } @@ -523,22 +601,21 @@ private Builder setTasks(List Builder addTask(String action, Request request, boolean stopped, - boolean removeOnCompletion, String executorNode) { + boolean removeOnCompletion, Assignment assignment) { changed = true; currentId++; - tasks.put(currentId, new PersistentTaskInProgress<>(currentId, action, request, stopped, removeOnCompletion, - executorNode)); + tasks.put(currentId, new PersistentTaskInProgress<>(currentId, action, request, stopped, removeOnCompletion, assignment)); return this; } /** * Reassigns the task to another node if the task exist */ - public Builder reassignTask(long taskId, String executorNode) { + public Builder reassignTask(long taskId, Assignment assignment) { PersistentTaskInProgress taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, executorNode)); + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, assignment)); } return this; } @@ -551,13 +628,13 @@ public Builder reassignTask(long taskId, String executorNode) { */ @SuppressWarnings("unchecked") public Builder assignTask(long taskId, - BiFunction executorNodeFunc) { + BiFunction executorNodeFunc) { PersistentTaskInProgress taskInProgress = (PersistentTaskInProgress) tasks.get(taskId); - if (taskInProgress != null && taskInProgress.getExecutorNode() == null) { // only assign unassigned tasks - String executorNode = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); - if (executorNode != null || taskInProgress.isStopped()) { + if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks + Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + if (assignment.isAssigned() || taskInProgress.isStopped()) { changed = true; - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, executorNode)); + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, assignment)); } } return this; @@ -568,12 +645,12 @@ public Builder assignTask(long taskId, */ @SuppressWarnings("unchecked") public Builder reassignTask(long taskId, - BiFunction executorNodeFunc) { + BiFunction executorNodeFunc) { PersistentTaskInProgress taskInProgress = (PersistentTaskInProgress) tasks.get(taskId); if (taskInProgress != null) { changed = true; - String executorNode = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, executorNode)); + Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, assignment)); } return this; } @@ -612,7 +689,7 @@ public Builder finishTask(long taskId) { if (taskInProgress.removeOnCompletion) { tasks.remove(taskId); } else { - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, true, null)); + tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, true, FINISHED_TASK_ASSIGNMENT)); } } return this; diff --git a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java index 9e58ee0896df1..833ae267216a4 100644 --- a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; import java.util.function.Predicate; import java.util.function.Supplier; @@ -56,15 +57,23 @@ protected TransportPersistentAction(Settings settings, String actionName, boolea persistentActionRegistry.registerPersistentAction(actionName, this); } + public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment"); + /** * Returns the node id where the request has to be executed, *

* The default implementation returns the least loaded data node */ - public DiscoveryNode executorNode(Request request, ClusterState clusterState) { - return selectLeastLoadedNode(clusterState, DiscoveryNode::isDataNode); + public Assignment getAssignment(Request request, ClusterState clusterState) { + DiscoveryNode discoveryNode = selectLeastLoadedNode(clusterState, DiscoveryNode::isDataNode); + if (discoveryNode == null) { + return NO_NODE_FOUND; + } else { + return new Assignment(discoveryNode.getId(), ""); + } } + /** * Finds the least loaded node that satisfies the selector criteria */ diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java index 7de67a22caefd..75a320fde4ebe 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.CompletionPersistentTaskAction.Response; +import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; import java.io.IOException; @@ -89,10 +90,12 @@ public void testStartTask() throws Exception { boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { - tasks.addTask("test_action", new TestRequest("other_" + i), false, true, "other_node_" + randomInt(nonLocalNodesCount)); + tasks.addTask("test_action", new TestRequest("other_" + i), false, true, + new Assignment("other_node_" + randomInt(nonLocalNodesCount), "test assignment on other node")); if (added == false && randomBoolean()) { added = true; - tasks.addTask("test", new TestRequest("this_param"), false, true, "this_node"); + tasks.addTask("test", new TestRequest("this_param"), false, true, + new Assignment("this_node", "test assignment on this node")); } } } @@ -302,7 +305,7 @@ private ClusterState addTask(ClusterSt PersistentTasksInProgress.Builder builder = PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, - builder.addTask(action, request, false, true, node).build())).build(); + builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build(); } private ClusterState reallocateTask(ClusterState state, long taskId, String node) { @@ -310,7 +313,7 @@ private ClusterState reallocateTask(ClusterState state, long taskId, String node PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); assertTrue(builder.hasTask(taskId)); return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, - builder.reassignTask(taskId, node).build())).build(); + builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build(); } private ClusterState removeTask(ClusterState state, long taskId) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java index fb6e884a1bfc2..32f43783bb4af 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; @@ -38,6 +39,7 @@ import java.util.List; import static java.util.Collections.emptyMap; +import static org.elasticsearch.persistent.TransportPersistentAction.NO_NODE_FOUND; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; @@ -46,7 +48,7 @@ public class PersistentTaskClusterServiceTests extends ESTestCase { public void testReassignmentRequired() { - int numberOfIterations = randomIntBetween(10, 100); + int numberOfIterations = randomIntBetween(1, 30); ClusterState clusterState = initialState(); for (int i = 0; i < numberOfIterations; i++) { boolean significant = randomBoolean(); @@ -58,14 +60,17 @@ public void testReassignmentRequired() { clusterState = insignificantChange(clusterState); } ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState); - assertThat(dumpEvent(event), significant, equalTo(PersistentTaskClusterService.reassignmentRequired(event, + assertThat(dumpEvent(event), PersistentTaskClusterService.reassignmentRequired(event, new PersistentTaskClusterService.ExecutorNodeDecider() { @Override - public String executorNode( + public Assignment getAssignment( String action, ClusterState currentState, Request request) { - return randomNode(currentState.nodes()); + if ("never_assign".equals(((TestRequest) request).getTestParam())) { + return NO_NODE_FOUND; + } + return randomNodeAssignment(currentState.nodes()); } - }))); + }), equalTo(significant)); } } @@ -138,23 +143,32 @@ public void testReassignTasks() { for (PersistentTaskInProgress task : tasksInProgress.tasks()) { if (task.isStopped()) { assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue()); + assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getAction())); } else { + // explanation should correspond to the action name switch (task.getAction()) { case "should_assign": assertThat(task.getExecutorNode(), notNullValue()); + assertThat(task.isAssigned(), equalTo(true)); if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { logger.info(clusterState.metaData().custom(PersistentTasksInProgress.TYPE).toString()); } assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); + assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); break; case "should_not_assign": assertThat(task.getExecutorNode(), nullValue()); + assertThat(task.isAssigned(), equalTo(false)); + assertThat(task.getAssignment().getExplanation(), equalTo("no appropriate nodes found for the assignment")); break; case "assign_one": - if (task.getExecutorNode() != null) { + if (task.isAssigned()) { assignOneCount++; assertThat("more than one assign_one tasks are assigned", assignOneCount, lessThanOrEqualTo(1)); + assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + } else { + assertThat(task.getAssignment().getExplanation(), equalTo("only one task can be assigned at a time")); } break; default: @@ -175,14 +189,14 @@ private ClusterState reassign(ClusterState clusterState) { return PersistentTaskClusterService.reassignTasks(clusterState, logger, new PersistentTaskClusterService.ExecutorNodeDecider() { @Override - public String executorNode( + public Assignment getAssignment( String action, ClusterState currentState, Request request) { TestRequest testRequest = (TestRequest) request; switch (testRequest.getTestParam()) { case "assign_me": - return randomNode(currentState.nodes()); + return randomNodeAssignment(currentState.nodes()); case "dont_assign_me": - return null; + return NO_NODE_FOUND; case "fail_me_if_called": fail("the decision decider shouldn't be called on this task"); return null; @@ -191,33 +205,37 @@ public String executorNode( default: fail("unknown param " + testRequest.getTestParam()); } - return null; + return NO_NODE_FOUND; } }); } - private String assignOnlyOneTaskAtATime(ClusterState clusterState) { + private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { DiscoveryNodes nodes = clusterState.nodes(); PersistentTasksInProgress tasksInProgress = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); if (tasksInProgress.findTasks("assign_one", task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) { - return randomNode(clusterState.nodes()); + return randomNodeAssignment(clusterState.nodes()); } else { - return null; + return new Assignment(null, "only one task can be assigned at a time"); } } - private String randomNode(DiscoveryNodes nodes) { + private Assignment randomNodeAssignment(DiscoveryNodes nodes) { if (nodes.getNodes().isEmpty()) { - return null; + return NO_NODE_FOUND; } List nodeList = new ArrayList<>(); for (ObjectCursor node : nodes.getNodes().keys()) { nodeList.add(node.value); } - return randomFrom(nodeList); - + String node = randomFrom(nodeList); + if (node != null) { + return new Assignment(node, "test assignment"); + } else { + return NO_NODE_FOUND; + } } private String dumpEvent(ClusterChangedEvent event) { @@ -232,10 +250,8 @@ private ClusterState significantChange(ClusterState clusterState) { PersistentTasksInProgress tasks = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); if (tasks != null) { if (randomBoolean()) { - // - boolean removedNode = false; for (PersistentTaskInProgress task : tasks.tasks()) { - if (task.getExecutorNode() != null && clusterState.nodes().nodeExists(task.getExecutorNode())) { + if (task.isAssigned() && clusterState.nodes().nodeExists(task.getExecutorNode())) { logger.info("removed node {}", task.getExecutorNode()); builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(task.getExecutorNode())); return builder.build(); @@ -245,11 +261,18 @@ private ClusterState significantChange(ClusterState clusterState) { } boolean tasksOrNodesChanged = false; // add a new unassigned task - if (hasUnassigned(tasks, clusterState.nodes()) == false) { + if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { // we don't have any unassigned tasks - add some - logger.info("added random task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), null, false); - tasksOrNodesChanged = true; + if (randomBoolean()) { + logger.info("added random task"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), null, false); + tasksOrNodesChanged = true; + } else { + logger.info("added unassignable task with custom assignment message"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), + new Assignment(null, "change me"), "never_assign", false); + tasksOrNodesChanged = true; + } } // add a node if there are unassigned tasks if (clusterState.nodes().getNodes().isEmpty()) { @@ -269,29 +292,58 @@ private ClusterState significantChange(ClusterState clusterState) { return builder.build(); } + private PersistentTasksInProgress removeTasksWithChangingAssignment(PersistentTasksInProgress tasks) { + if (tasks != null) { + boolean changed = false; + PersistentTasksInProgress.Builder tasksBuilder = PersistentTasksInProgress.builder(tasks); + for (PersistentTaskInProgress task : tasks.tasks()) { + // Remove all unassigned tasks that cause changing assignments they might trigger a significant change + if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) && + "change me".equals(task.getAssignment().getExplanation())) { + logger.info("removed task with changing assignment {}", task.getId()); + tasksBuilder.removeTask(task.getId()); + changed = true; + } + } + if (changed) { + return tasksBuilder.build(); + } + } + return tasks; + } + private ClusterState insignificantChange(ClusterState clusterState) { ClusterState.Builder builder = ClusterState.builder(clusterState); PersistentTasksInProgress tasks = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + tasks = removeTasksWithChangingAssignment(tasks); + PersistentTasksInProgress.Builder tasksBuilder = PersistentTasksInProgress.builder(tasks); + if (randomBoolean()) { - if (hasUnassigned(tasks, clusterState.nodes()) == false) { + if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { // we don't have any unassigned tasks - adding a node or changing a routing table shouldn't affect anything if (randomBoolean()) { logger.info("added random node"); builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAsciiOfLength(10)))); } + if (randomBoolean()) { + logger.info("added random unassignable task"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), tasksBuilder, NO_NODE_FOUND, "never_assign", false); + return builder.build(); + } if (randomBoolean()) { // add unassigned task in stopped state logger.info("added random stopped task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), null, true); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), tasksBuilder, null, true); return builder.build(); } else { logger.info("changed routing table"); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); + metaData.putCustom(PersistentTasksInProgress.TYPE, tasksBuilder.build()); RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); changeRoutingTable(metaData, routingTable); builder.metaData(metaData).routingTable(routingTable.build()); + return builder.build(); } - return builder.build(); } } if (randomBoolean()) { @@ -318,10 +370,9 @@ private ClusterState insignificantChange(ClusterState clusterState) { } } logger.info("removed all unassigned tasks and changed routing table"); - PersistentTasksInProgress.Builder tasksBuilder = PersistentTasksInProgress.builder(tasks); if (tasks != null) { for (PersistentTaskInProgress task : tasks.tasks()) { - if (task.getExecutorNode() == null) { + if (task.getExecutorNode() == null || "never_assign".equals(((TestRequest) task.getRequest()).getTestParam())) { tasksBuilder.removeTask(task.getId()); } } @@ -337,13 +388,19 @@ private ClusterState insignificantChange(ClusterState clusterState) { return builder.metaData(metaData).build(); } - private boolean hasUnassigned(PersistentTasksInProgress tasks, DiscoveryNodes discoveryNodes) { + private boolean hasAssignableTasks(PersistentTasksInProgress tasks, DiscoveryNodes discoveryNodes) { if (tasks == null || tasks.tasks().isEmpty()) { return false; } - return tasks.tasks().stream().anyMatch(task -> - task.isStopped() == false && - (task.getExecutorNode() == null || discoveryNodes.nodeExists(task.getExecutorNode()))); + return tasks.tasks().stream().anyMatch(task -> { + if (task.isStopped()) { + return false; + } + if (task.getExecutorNode() == null || discoveryNodes.nodeExists(task.getExecutorNode())) { + return "never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) == false; + } + return false; + }); } private boolean hasTasksAssignedTo(PersistentTasksInProgress tasks, String nodeId) { @@ -353,14 +410,20 @@ private boolean hasTasksAssignedTo(PersistentTasksInProgress tasks, String nodeI private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, MetaData.Builder metaData, PersistentTasksInProgress.Builder tasks, - String node, - boolean stopped) { + String node, boolean stopped) { + return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)), + randomAsciiOfLength(10), stopped); + } + + private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, + MetaData.Builder metaData, PersistentTasksInProgress.Builder tasks, + Assignment assignment, String param, boolean stopped) { return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksInProgress.TYPE, - tasks.addTask(randomAsciiOfLength(10), new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), node).build())); + tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build())); } private void addTask(PersistentTasksInProgress.Builder tasks, String action, String param, String node, boolean stopped) { - tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), node); + tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action)); } private DiscoveryNode newNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java index 896d749772a54..60776c512562b 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractDiffableSerializationTestCase; +import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; import org.elasticsearch.persistent.PersistentTasksInProgress.Builder; import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; @@ -47,6 +48,7 @@ import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; +import static org.elasticsearch.persistent.TransportPersistentAction.NO_NODE_FOUND; public class PersistentTasksInProgressTests extends AbstractDiffableSerializationTestCase { @@ -55,8 +57,9 @@ protected PersistentTasksInProgress createTestInstance() { int numberOfTasks = randomInt(10); PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder(); for (int i = 0; i < numberOfTasks; i++) { + boolean stopped = randomBoolean(); tasks.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), - randomBoolean(), randomBoolean(), randomAsciiOfLength(10)); + stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment()); if (randomBoolean()) { // From time to time update status tasks.updateTaskStatus(tasks.getCurrentId(), new Status(randomAsciiOfLength(10))); @@ -92,7 +95,7 @@ protected Custom makeTestChanges(Custom testInstance) { if (tasksInProgress.tasks().isEmpty()) { addRandomTask(builder); } else { - builder.reassignTask(pickRandomTask(tasksInProgress), randomAsciiOfLength(10)); + builder.reassignTask(pickRandomTask(tasksInProgress), randomAssignment()); } break; case 2: @@ -146,8 +149,9 @@ protected XContentBuilder toXContent(Custom instance, XContentType contentType, } private Builder addRandomTask(Builder builder) { - builder.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), - randomBoolean(), randomBoolean(), randomAsciiOfLength(10)); + boolean stopped = randomBoolean(); + builder.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), + stopped ? new Assignment(null, "stopped") : randomAssignment()); return builder; } @@ -223,9 +227,9 @@ public void testBuilder() { changed = true; } if (randomBoolean()) { - builder.reassignTask(lastKnownTask, randomAsciiOfLength(10)); + builder.reassignTask(lastKnownTask, randomAssignment()); } else { - builder.reassignTask(lastKnownTask, (s, request) -> randomAsciiOfLength(10)); + builder.reassignTask(lastKnownTask, (s, request) -> randomAssignment()); } break; case 2: @@ -233,22 +237,23 @@ public void testBuilder() { PersistentTaskInProgress task = builder.build().getTask(lastKnownTask); if (randomBoolean()) { // Trying to reassign to the same node - builder.assignTask(lastKnownTask, (s, request) -> task.getExecutorNode()); + builder.assignTask(lastKnownTask, (s, request) -> task.getAssignment()); // should change if the task was stopped AND unassigned if (task.getExecutorNode() == null && task.isStopped()) { changed = true; } } else { // Trying to reassign to a different node - builder.assignTask(lastKnownTask, (s, request) -> randomAsciiOfLength(10)); - // should change if the task was unassigned - if (task.getExecutorNode() == null) { + Assignment randomAssignment = randomAssignment(); + builder.assignTask(lastKnownTask, (s, request) -> randomAssignment); + // should change if the task was unassigned and was reassigned to a different node or started + if ((task.isAssigned() == false && randomAssignment.isAssigned()) || task.isStopped()) { changed = true; } } } else { // task doesn't exist - shouldn't change - builder.assignTask(lastKnownTask, (s, request) -> randomAsciiOfLength(10)); + builder.assignTask(lastKnownTask, (s, request) -> randomAssignment()); } break; case 3: @@ -277,4 +282,14 @@ public void testBuilder() { } + private Assignment randomAssignment() { + if (randomBoolean()) { + if (randomBoolean()) { + return NO_NODE_FOUND; + } else { + return new Assignment(null, randomAsciiOfLength(10)); + } + } + return new Assignment(randomAsciiOfLength(10), randomAsciiOfLength(10)); + } } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java index bc8651dad24fb..2299bace1667f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java @@ -63,6 +63,7 @@ import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; import java.io.IOException; import java.util.ArrayList; @@ -365,12 +366,18 @@ public TransportTestPersistentAction(Settings settings, ThreadPool threadPool, T } @Override - public DiscoveryNode executorNode(TestRequest request, ClusterState clusterState) { + public Assignment getAssignment(TestRequest request, ClusterState clusterState) { if (request.getExecutorNodeAttr() == null) { - return super.executorNode(request, clusterState); + return super.getAssignment(request, clusterState); } else { - return selectLeastLoadedNode(clusterState, + DiscoveryNode executorNode = selectLeastLoadedNode(clusterState, discoveryNode -> request.getExecutorNodeAttr().equals(discoveryNode.getAttributes().get("test_attr"))); + if (executorNode != null) { + return new Assignment(executorNode.getId(), "test assignment"); + } else { + return NO_NODE_FOUND; + } + } } From b33fc05492656a73f1b6f7b8094945c50a240db4 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Feb 2017 17:28:33 -0500 Subject: [PATCH 10/50] Request and Status in Persistent Tasks should be serialized using their writable names Refactors xcontent serialization of Request and Status to use their writable names instead of action name. That simplifies the parsing logic, allows reuse of the same status object for multiple actions and is consistent with how named objects in xcontent are used. --- .../persistent/PersistentTasksInProgress.java | 79 +++++++------------ .../PersistentTasksInProgressTests.java | 3 +- 2 files changed, 31 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java index 9dcea474d982f..e980e65eaa6fa 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java @@ -71,16 +71,18 @@ public PersistentTasksInProgress(long currentId, Map PERSISTENT_TASKS_IN_PROGRESS_PARSER = new ObjectParser<>(TYPE, - Builder::new); + private static final ObjectParser PERSISTENT_TASKS_IN_PROGRESS_PARSER = new ObjectParser<>(TYPE, Builder::new); - public static final ObjectParser, Void> PERSISTENT_TASK_IN_PROGRESS_PARSER = + private static final ObjectParser, Void> PERSISTENT_TASK_IN_PROGRESS_PARSER = new ObjectParser<>("running_tasks", TaskBuilder::new); public static final ConstructingObjectParser ASSIGNMENT_PARSER = new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); - public static final NamedObjectParser, Void> ACTION_PARSER; + private static final NamedObjectParser REQUEST_PARSER = + (XContentParser p, Void c, String name) -> p.namedObject(PersistentActionRequest.class, name, null); + private static final NamedObjectParser STATUS_PARSER = + (XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null); static { // Tasks parser initialization @@ -88,13 +90,6 @@ public PersistentTasksInProgress(long currentId, Map, String> parser = new ObjectParser<>("named"); - parser.declareObject(ActionDescriptionBuilder::setRequest, - (p, c) -> p.namedObject(PersistentActionRequest.class, c, null), new ParseField("request")); - parser.declareObject(ActionDescriptionBuilder::setStatus, - (p, c) -> p.namedObject(Status.class, c, null), new ParseField("status")); - ACTION_PARSER = (XContentParser p, Void c, String name) -> parser.parse(p, new ActionDescriptionBuilder<>(name), name); // Assignment parser ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("executor_node")); @@ -102,45 +97,30 @@ public PersistentTasksInProgress(long currentId, Map taskBuilder, List> objects) -> { + (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { - throw new IllegalArgumentException("only one action description per task is allowed"); + throw new IllegalArgumentException("only one action request per task is allowed"); } - ActionDescriptionBuilder builder = objects.get(0); - taskBuilder.setAction(builder.action); - taskBuilder.setRequest(builder.request); - taskBuilder.setStatus(builder.status); - }, ACTION_PARSER, new ParseField("action")); - PERSISTENT_TASK_IN_PROGRESS_PARSER.declareObject(TaskBuilder::setAssignment, ASSIGNMENT_PARSER, new ParseField("assignment")); - PERSISTENT_TASK_IN_PROGRESS_PARSER.declareLong(TaskBuilder::setAllocationIdOnLastStatusUpdate, - new ParseField("allocation_id_on_last_status_update")); - } - - /** - * Private builder used in XContent parser - */ - private static class ActionDescriptionBuilder { - private final String action; - private Request request; - private Status status; + taskBuilder.setRequest(objects.get(0)); + }, REQUEST_PARSER, new ParseField("request")); - private ActionDescriptionBuilder(String action) { - this.action = action; - } + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareNamedObjects( + (TaskBuilder taskBuilder, List objects) -> { + if (objects.size() != 1) { + throw new IllegalArgumentException("only one status per task is allowed"); + } + taskBuilder.setStatus(objects.get(0)); + }, STATUS_PARSER, new ParseField("status")); - private ActionDescriptionBuilder setRequest(Request request) { - this.request = request; - return this; - } - private ActionDescriptionBuilder setStatus(Status status) { - this.status = status; - return this; - } + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareObject(TaskBuilder::setAssignment, ASSIGNMENT_PARSER, new ParseField("assignment")); + PERSISTENT_TASK_IN_PROGRESS_PARSER.declareLong(TaskBuilder::setAllocationIdOnLastStatusUpdate, + new ParseField("allocation_id_on_last_status_update")); } public Collection> tasks() { @@ -417,19 +397,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field("id", id); - builder.startObject("action"); + builder.field("action", action); + builder.startObject("request"); { - builder.startObject(action); + builder.field(request.getWriteableName(), request, params); + } + builder.endObject(); + if (status != null) { + builder.startObject("status"); { - builder.field("request"); - request.toXContent(builder, params); - if (status != null) { - builder.field("status", status, params); - } + builder.field(status.getWriteableName(), status, params); } builder.endObject(); } - builder.endObject(); + if (API_CONTEXT.equals(params.param(MetaData.CONTEXT_MODE_PARAM, API_CONTEXT))) { // These are transient values that shouldn't be persisted to gateway cluster state or snapshot builder.field("allocation_id", allocationId); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java index 60776c512562b..0f55fdc4bfedc 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java @@ -164,8 +164,7 @@ protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(Arrays.asList( new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), TestRequest::fromXContent), - new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(TestPersistentAction.NAME), - Status::fromXContent) + new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) )); } From 810d9335c0630746a9e807e20d53e02299d88742 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Feb 2017 11:37:42 -0700 Subject: [PATCH 11/50] Simplify names of PersistentTasks-related classes PersistentTask -> NodePersistentTask PersistentTasksInProgress -> PersistentTasks PersistentTaskInProgress -> PersistentTask --- ...stentTask.java => NodePersistentTask.java} | 6 +- .../PersistentActionCoordinator.java | 20 +-- .../persistent/PersistentActionExecutor.java | 2 +- .../persistent/PersistentActionRequest.java | 2 +- .../PersistentTaskClusterService.java | 35 +++--- ...ksInProgress.java => PersistentTasks.java} | 116 +++++++++--------- .../persistent/TransportPersistentAction.java | 12 +- .../persistent/package-info.java | 6 +- .../PersistentActionCoordinatorTests.java | 32 ++--- .../PersistentActionFullRestartIT.java | 18 +-- .../persistent/PersistentActionIT.java | 20 +-- .../PersistentTaskClusterServiceTests.java | 72 +++++------ ...ssTests.java => PersistentTasksTests.java} | 46 +++---- .../TestPersistentActionPlugin.java | 14 +-- 14 files changed, 198 insertions(+), 203 deletions(-) rename server/src/main/java/org/elasticsearch/persistent/{PersistentTask.java => NodePersistentTask.java} (90%) rename server/src/main/java/org/elasticsearch/persistent/{PersistentTasksInProgress.java => PersistentTasks.java} (80%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentTasksInProgressTests.java => PersistentTasksTests.java} (86%) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTask.java similarity index 90% rename from server/src/main/java/org/elasticsearch/persistent/PersistentTask.java rename to server/src/main/java/org/elasticsearch/persistent/NodePersistentTask.java index 973b7a8836212..6721775125caf 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTask.java @@ -23,14 +23,14 @@ import org.elasticsearch.tasks.TaskId; /** - * Task that returns additional state information + * Represents a executor node operation that corresponds to a persistent task */ -public class PersistentTask extends CancellableTask { +public class NodePersistentTask extends CancellableTask { private Provider statusProvider; private long persistentTaskId; - public PersistentTask(long id, String type, String action, String description, TaskId parentTask) { + public NodePersistentTask(long id, String type, String action, String description, TaskId parentTask) { super(id, type, action, description, parentTask); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java index ccf3a9dc35d1d..427fa129f89f1 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java @@ -37,7 +37,7 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.PersistentTasks.PersistentTask; import java.io.IOException; import java.util.HashMap; @@ -75,15 +75,15 @@ public PersistentActionCoordinator(Settings settings, @Override public void clusterChanged(ClusterChangedEvent event) { - PersistentTasksInProgress tasks = event.state().getMetaData().custom(PersistentTasksInProgress.TYPE); - PersistentTasksInProgress previousTasks = event.previousState().getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasks previousTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { // We have some changes let's check if they are related to our node String localNodeId = event.state().getNodes().getLocalNodeId(); Set notVisitedTasks = new HashSet<>(runningTasks.keySet()); if (tasks != null) { - for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { + for (PersistentTask taskInProgress : tasks.tasks()) { if (localNodeId.equals(taskInProgress.getExecutorNode())) { PersistentTaskId persistentTaskId = new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()); RunningPersistentTask persistentTask = runningTasks.get(persistentTaskId); @@ -124,10 +124,10 @@ public void clusterChanged(ClusterChangedEvent event) { } - private void startTask(PersistentTaskInProgress taskInProgress) { + private void startTask(PersistentTask taskInProgress) { PersistentActionRegistry.PersistentActionHolder holder = persistentActionRegistry.getPersistentActionHolderSafe(taskInProgress.getAction()); - PersistentTask task = (PersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]", + NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]", taskInProgress.getRequest()); boolean processed = false; try { @@ -296,23 +296,23 @@ public int hashCode() { } private static class RunningPersistentTask implements Provider { - private final PersistentTask task; + private final NodePersistentTask task; private final long id; private final AtomicReference state; @Nullable private Exception failure; - RunningPersistentTask(PersistentTask task, long id) { + RunningPersistentTask(NodePersistentTask task, long id) { this(task, id, State.STARTED); } - RunningPersistentTask(PersistentTask task, long id, State state) { + RunningPersistentTask(NodePersistentTask task, long id, State state) { this.task = task; this.id = id; this.state = new AtomicReference<>(state); } - public PersistentTask getTask() { + public NodePersistentTask getTask() { return task; } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java index de6c6ebd0680d..418ce8a36e325 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java @@ -34,7 +34,7 @@ public PersistentActionExecutor(ThreadPool threadPool) { } public void executeAction(Request request, - PersistentTask task, + NodePersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, ActionListener listener) { threadPool.executor(holder.getExecutor()).execute(new AbstractRunnable() { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java index d1e6bcfe6ec20..dfd1da1ea438d 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java @@ -30,6 +30,6 @@ public abstract class PersistentActionRequest extends ActionRequest implements NamedWriteable, ToXContent { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId) { - return new PersistentTask(id, type, action, getDescription(), parentTaskId); + return new NodePersistentTask(id, type, action, getDescription(), parentTaskId); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java index 54e8391f4d7e5..372fd05ed1e14 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java @@ -27,15 +27,14 @@ import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.PersistentTasks.Assignment; +import org.elasticsearch.persistent.PersistentTasks.PersistentTask; import java.util.Objects; @@ -70,7 +69,7 @@ public void createPersistentTask(Strin public ClusterState execute(ClusterState currentState) throws Exception { final Assignment assignment; if (stopped) { - assignment = PersistentTasksInProgress.FINISHED_TASK_ASSIGNMENT; // the task is stopped no need to assign it anywhere + assignment = PersistentTasks.FINISHED_TASK_ASSIGNMENT; // the task is stopped no need to assign it anywhere } else { assignment = getAssignement(action, currentState, request); } @@ -85,7 +84,7 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { listener.onResponse( - ((PersistentTasksInProgress) newState.getMetaData().custom(PersistentTasksInProgress.TYPE)).getCurrentId()); + ((PersistentTasks) newState.getMetaData().custom(PersistentTasks.TYPE)).getCurrentId()); } }); } @@ -109,7 +108,7 @@ public void completeOrRestartPersistentTask(long id, Exception failure, ActionLi clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + PersistentTasks.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { if (failure != null) { // If the task failed - we need to restart it on another node, otherwise we just remove it @@ -147,7 +146,7 @@ public void startPersistentTask(long id, ActionListener listener) { clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + PersistentTasks.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress .assignTask(id, (action, request) -> getAssignement(action, currentState, request))); @@ -178,7 +177,7 @@ public void removePersistentTask(long id, ActionListener listener) { clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + PersistentTasks.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress.removeTask(id)); } else { @@ -209,7 +208,7 @@ public void updatePersistentTaskStatus(long id, Task.Status status, ActionListen clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksInProgress.Builder tasksInProgress = builder(currentState); + PersistentTasks.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress.updateTaskStatus(id, status)); } else { @@ -253,15 +252,15 @@ interface ExecutorNodeDecider { } static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { - PersistentTasksInProgress tasks = event.state().getMetaData().custom(PersistentTasksInProgress.TYPE); - PersistentTasksInProgress prevTasks = event.previousState().getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasks prevTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); if (tasks != null && (Objects.equals(tasks, prevTasks) == false || event.nodesChanged() || event.routingTableChanged() || event.previousState().nodes().isLocalNodeElectedMaster() == false)) { // We need to check if removed nodes were running any of the tasks and reassign them boolean reassignmentRequired = false; - for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { + for (PersistentTask taskInProgress : tasks.tasks()) { if (taskInProgress.needsReassignment(event.state().nodes())) { // there is an unassigned task or task with a disappeared node - we need to try assigning it if (Objects.equals(taskInProgress.getAssignment(), @@ -301,13 +300,13 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) { - PersistentTasksInProgress tasks = currentState.getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasks = currentState.getMetaData().custom(PersistentTasks.TYPE); ClusterState clusterState = currentState; DiscoveryNodes nodes = currentState.nodes(); if (tasks != null) { logger.trace("reassigning {} persistent tasks", tasks.tasks().size()); // We need to check if removed nodes were running any of the tasks and reassign them - for (PersistentTaskInProgress task : tasks.tasks()) { + for (PersistentTask task : tasks.tasks()) { if (task.needsReassignment(nodes)) { // there is an unassigned task - we need to try assigning it Assignment assignment = decider.getAssignment(task.getAction(), clusterState, task.getRequest()); @@ -330,14 +329,14 @@ static ClusterState reassignTasks(ClusterState currentState, Logger logger, Exec return clusterState; } - private static PersistentTasksInProgress.Builder builder(ClusterState currentState) { - return PersistentTasksInProgress.builder(currentState.getMetaData().custom(PersistentTasksInProgress.TYPE)); + private static PersistentTasks.Builder builder(ClusterState currentState) { + return PersistentTasks.builder(currentState.getMetaData().custom(PersistentTasks.TYPE)); } - private static ClusterState update(ClusterState currentState, PersistentTasksInProgress.Builder tasksInProgress) { + private static ClusterState update(ClusterState currentState, PersistentTasks.Builder tasksInProgress) { if (tasksInProgress.isChanged()) { return ClusterState.builder(currentState).metaData( - MetaData.builder(currentState.metaData()).putCustom(PersistentTasksInProgress.TYPE, tasksInProgress.build()) + MetaData.builder(currentState.metaData()).putCustom(PersistentTasks.TYPE, tasksInProgress.build()) ).build(); } else { return currentState; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasks.java similarity index 80% rename from server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTasks.java index e980e65eaa6fa..855178c760e26 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksInProgress.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasks.java @@ -56,25 +56,25 @@ /** * A cluster state record that contains a list of all running persistent tasks */ -public final class PersistentTasksInProgress extends AbstractNamedDiffable implements MetaData.Custom { +public final class PersistentTasks extends AbstractNamedDiffable implements MetaData.Custom { public static final String TYPE = "persistent_tasks"; private static final String API_CONTEXT = MetaData.XContentContext.API.toString(); // TODO: Implement custom Diff for tasks - private final Map> tasks; + private final Map> tasks; private final long currentId; - public PersistentTasksInProgress(long currentId, Map> tasks) { + public PersistentTasks(long currentId, Map> tasks) { this.currentId = currentId; this.tasks = tasks; } - private static final ObjectParser PERSISTENT_TASKS_IN_PROGRESS_PARSER = new ObjectParser<>(TYPE, Builder::new); + private static final ObjectParser PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new); - private static final ObjectParser, Void> PERSISTENT_TASK_IN_PROGRESS_PARSER = - new ObjectParser<>("running_tasks", TaskBuilder::new); + private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = + new ObjectParser<>("tasks", TaskBuilder::new); public static final ConstructingObjectParser ASSIGNMENT_PARSER = new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); @@ -86,22 +86,20 @@ public PersistentTasksInProgress(long currentId, Map taskBuilder, List objects) -> { if (objects.size() != 1) { throw new IllegalArgumentException("only one action request per task is allowed"); @@ -109,7 +107,7 @@ public PersistentTasksInProgress(long currentId, Map taskBuilder, List objects) -> { if (objects.size() != 1) { throw new IllegalArgumentException("only one status per task is allowed"); @@ -118,31 +116,31 @@ public PersistentTasksInProgress(long currentId, Map> tasks() { + public Collection> tasks() { return this.tasks.values(); } - public Map> taskMap() { + public Map> taskMap() { return this.tasks; } - public PersistentTaskInProgress getTask(long id) { + public PersistentTask getTask(long id) { return this.tasks.get(id); } - public Collection> findTasks(String actionName, Predicate> predicate) { + public Collection> findTasks(String actionName, Predicate> predicate) { return this.tasks().stream() .filter(p -> actionName.equals(p.getAction())) .filter(predicate) .collect(Collectors.toList()); } - public boolean tasksExist(String actionName, Predicate> predicate) { + public boolean tasksExist(String actionName, Predicate> predicate) { return this.tasks().stream() .filter(p -> actionName.equals(p.getAction())) .anyMatch(predicate); @@ -152,7 +150,7 @@ public boolean tasksExist(String actionName, Predicate context() { return ALL_CONTEXTS; } - public static PersistentTasksInProgress fromXContent(XContentParser parser) throws IOException { - return PERSISTENT_TASKS_IN_PROGRESS_PARSER.parse(parser, null).build(); + public static PersistentTasks fromXContent(XContentParser parser) throws IOException { + return PERSISTENT_TASKS_PARSER.parse(parser, null).build(); } public static class Assignment { @@ -236,7 +234,7 @@ public String toString() { /** * A record that represents a single running persistent task */ - public static class PersistentTaskInProgress implements Writeable, ToXContent { + public static class PersistentTask implements Writeable, ToXContent { private final long id; private final long allocationId; private final String action; @@ -250,24 +248,22 @@ public static class PersistentTaskInProgress task, boolean stopped, Assignment assignment) { + public PersistentTask(PersistentTask task, boolean stopped, Assignment assignment) { this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status, assignment, task.allocationId); } - public PersistentTaskInProgress(PersistentTaskInProgress task, Status status) { + public PersistentTask(PersistentTask task, Status status) { this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, task.assignment, task.allocationId); } - private PersistentTaskInProgress(long id, long allocationId, String action, Request request, - boolean stopped, boolean removeOnCompletion, Status status, - Assignment assignment, Long allocationIdOnLastStatusUpdate) { + private PersistentTask(long id, long allocationId, String action, Request request, boolean stopped, boolean removeOnCompletion, + Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; this.action = action; @@ -282,7 +278,7 @@ private PersistentTaskInProgress(long id, long allocationId, String action, Requ } @SuppressWarnings("unchecked") - private PersistentTaskInProgress(StreamInput in) throws IOException { + private PersistentTask(StreamInput in) throws IOException { id = in.readLong(); allocationId = in.readLong(); action = in.readString(); @@ -312,7 +308,7 @@ public void writeTo(StreamOutput out) throws IOException { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - PersistentTaskInProgress that = (PersistentTaskInProgress) o; + PersistentTask that = (PersistentTask) o; return id == that.id && allocationId == that.allocationId && Objects.equals(action, that.action) && @@ -494,8 +490,8 @@ public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdO return this; } - public PersistentTaskInProgress build() { - return new PersistentTaskInProgress<>(id, allocationId, action, request, stopped, removeOnCompletion, status, + public PersistentTask build() { + return new PersistentTask<>(id, allocationId, action, request, stopped, removeOnCompletion, status, assignment, allocationIdOnLastStatusUpdate); } } @@ -505,9 +501,9 @@ public String getWriteableName() { return TYPE; } - public PersistentTasksInProgress(StreamInput in) throws IOException { + public PersistentTasks(StreamInput in) throws IOException { currentId = in.readLong(); - tasks = in.readMap(StreamInput::readLong, PersistentTaskInProgress::new); + tasks = in.readMap(StreamInput::readLong, PersistentTask::new); } @Override @@ -530,8 +526,8 @@ public long getCurrentId() { @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.field("current_id", currentId); - builder.startArray("running_tasks"); - for (PersistentTaskInProgress entry : tasks.values()) { + builder.startArray("tasks"); + for (PersistentTask entry : tasks.values()) { entry.toXContent(builder, params); } builder.endArray(); @@ -542,19 +538,19 @@ public static Builder builder() { return new Builder(); } - public static Builder builder(PersistentTasksInProgress tasks) { + public static Builder builder(PersistentTasks tasks) { return new Builder(tasks); } public static class Builder { - private final Map> tasks = new HashMap<>(); + private final Map> tasks = new HashMap<>(); private long currentId; private boolean changed; public Builder() { } - public Builder(PersistentTasksInProgress tasksInProgress) { + public Builder(PersistentTasks tasksInProgress) { if (tasksInProgress != null) { tasks.putAll(tasksInProgress.tasks); currentId = tasksInProgress.currentId; @@ -570,7 +566,7 @@ private Builder setCurrentId(long currentId) { private Builder setTasks(List> tasks) { for (TaskBuilder builder : tasks) { - PersistentTaskInProgress task = builder.build(); + PersistentTask task = builder.build(); this.tasks.put(task.getId(), task); } return this; @@ -585,7 +581,7 @@ public Builder addTask(String action, boolean removeOnCompletion, Assignment assignment) { changed = true; currentId++; - tasks.put(currentId, new PersistentTaskInProgress<>(currentId, action, request, stopped, removeOnCompletion, assignment)); + tasks.put(currentId, new PersistentTask<>(currentId, action, request, stopped, removeOnCompletion, assignment)); return this; } @@ -593,10 +589,10 @@ public Builder addTask(String action, * Reassigns the task to another node if the task exist */ public Builder reassignTask(long taskId, Assignment assignment) { - PersistentTaskInProgress taskInProgress = tasks.get(taskId); + PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, assignment)); + tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); } return this; } @@ -610,12 +606,12 @@ public Builder reassignTask(long taskId, Assignment assignment) { @SuppressWarnings("unchecked") public Builder assignTask(long taskId, BiFunction executorNodeFunc) { - PersistentTaskInProgress taskInProgress = (PersistentTaskInProgress) tasks.get(taskId); + PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); if (assignment.isAssigned() || taskInProgress.isStopped()) { changed = true; - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, assignment)); + tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); } } return this; @@ -627,11 +623,11 @@ public Builder assignTask(long taskId, @SuppressWarnings("unchecked") public Builder reassignTask(long taskId, BiFunction executorNodeFunc) { - PersistentTaskInProgress taskInProgress = (PersistentTaskInProgress) tasks.get(taskId); + PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); if (taskInProgress != null) { changed = true; Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, false, assignment)); + tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); } return this; } @@ -640,10 +636,10 @@ public Builder reassignTask(long taskI * Updates the task status if the task exist */ public Builder updateTaskStatus(long taskId, Status status) { - PersistentTaskInProgress taskInProgress = tasks.get(taskId); + PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, status)); + tasks.put(taskId, new PersistentTask<>(taskInProgress, status)); } return this; } @@ -664,13 +660,13 @@ public Builder removeTask(long taskId) { * If the task is marked with removeOnCompletion flag, it is removed from the list, otherwise it is stopped. */ public Builder finishTask(long taskId) { - PersistentTaskInProgress taskInProgress = tasks.get(taskId); + PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; if (taskInProgress.removeOnCompletion) { tasks.remove(taskId); } else { - tasks.put(taskId, new PersistentTaskInProgress<>(taskInProgress, true, FINISHED_TASK_ASSIGNMENT)); + tasks.put(taskId, new PersistentTask<>(taskInProgress, true, FINISHED_TASK_ASSIGNMENT)); } } return this; @@ -697,8 +693,8 @@ public boolean isChanged() { return changed; } - public PersistentTasksInProgress build() { - return new PersistentTasksInProgress(currentId, Collections.unmodifiableMap(tasks)); + public PersistentTasks build() { + return new PersistentTasks(currentId, Collections.unmodifiableMap(tasks)); } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java index 833ae267216a4..a4c36884108af 100644 --- a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java @@ -30,7 +30,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; +import org.elasticsearch.persistent.PersistentTasks.Assignment; import java.util.function.Predicate; import java.util.function.Supplier; @@ -80,14 +80,14 @@ public Assignment getAssignment(Request request, ClusterState clusterState) { protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate selector) { long minLoad = Long.MAX_VALUE; DiscoveryNode minLoadedNode = null; - PersistentTasksInProgress persistentTasksInProgress = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); for (DiscoveryNode node : clusterState.getNodes()) { if (selector.test(node)) { - if (persistentTasksInProgress == null) { + if (persistentTasks == null) { // We don't have any task running yet, pick the first available node return node; } - long numberOfTasks = persistentTasksInProgress.getNumberOfTasksOnNode(node.getId(), actionName); + long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), actionName); if (minLoad > numberOfTasks) { minLoad = numberOfTasks; minLoadedNode = node; @@ -117,7 +117,7 @@ protected void doExecute(Request request, ActionListener listener) { + protected void updatePersistentTaskStatus(NodePersistentTask task, Task.Status status, ActionListener listener) { persistentActionService.updateStatus(task.getPersistentTaskId(), status, new ActionListener() { @Override @@ -139,7 +139,7 @@ public void onFailure(Exception e) { * possibly on a different node. If listener.onResponse() is called, the task is considered to be successfully * completed and will be removed from the cluster state and not restarted. */ - protected abstract void nodeOperation(PersistentTask task, Request request, ActionListener listener); + protected abstract void nodeOperation(NodePersistentTask task, Request request, ActionListener listener); public String getExecutor() { return executor; diff --git a/server/src/main/java/org/elasticsearch/persistent/package-info.java b/server/src/main/java/org/elasticsearch/persistent/package-info.java index 9994dedeab80b..ef8210f6db9cb 100644 --- a/server/src/main/java/org/elasticsearch/persistent/package-info.java +++ b/server/src/main/java/org/elasticsearch/persistent/package-info.java @@ -29,7 +29,7 @@ * {@link org.elasticsearch.persistent.PersistentTaskClusterService} to update cluster state with the record about running persistent * task. *

- * 2. The master node updates the {@link org.elasticsearch.persistent.PersistentTasksInProgress} in the cluster state to indicate that + * 2. The master node updates the {@link org.elasticsearch.persistent.PersistentTasks} in the cluster state to indicate that * there is a new persistent action * running in the system. *

@@ -37,11 +37,11 @@ * the cluster state and starts execution of all new actions assigned to the node it is running on. *

* 4. If the action fails to start on the node, the {@link org.elasticsearch.persistent.PersistentActionCoordinator} uses the - * {@link org.elasticsearch.persistent.PersistentTasksInProgress} to notify the + * {@link org.elasticsearch.persistent.PersistentTasks} to notify the * {@link org.elasticsearch.persistent.PersistentActionService}, which reassigns the action to another node in the cluster. *

* 5. If action finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the - * cluster state. + * cluster state unless . *

* 6. The {@link org.elasticsearch.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent action. */ diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java index 75a320fde4ebe..10ebb4fa2703a 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java @@ -37,7 +37,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.CompletionPersistentTaskAction.Response; -import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; +import org.elasticsearch.persistent.PersistentTasks.Assignment; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; import java.io.IOException; @@ -86,7 +86,7 @@ public void testStartTask() throws Exception { ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); - PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder(); + PersistentTasks.Builder tasks = PersistentTasks.builder(); boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { @@ -105,7 +105,7 @@ public void testStartTask() throws Exception { } MetaData.Builder metaData = MetaData.builder(state.metaData()); - metaData.putCustom(PersistentTasksInProgress.TYPE, tasks.build()); + metaData.putCustom(PersistentTasks.TYPE, tasks.build()); ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); @@ -302,36 +302,36 @@ public void sendCompletionNotification(long taskId, Exception failure, ActionLis private ClusterState addTask(ClusterState state, String action, Request request, String node) { - PersistentTasksInProgress.Builder builder = - PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, + PersistentTasks.Builder builder = + PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build(); } private ClusterState reallocateTask(ClusterState state, long taskId, String node) { - PersistentTasksInProgress.Builder builder = - PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); + PersistentTasks.Builder builder = + PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); assertTrue(builder.hasTask(taskId)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build(); } private ClusterState removeTask(ClusterState state, long taskId) { - PersistentTasksInProgress.Builder builder = - PersistentTasksInProgress.builder(state.getMetaData().custom(PersistentTasksInProgress.TYPE)); + PersistentTasks.Builder builder = + PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); assertTrue(builder.hasTask(taskId)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksInProgress.TYPE, + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, builder.removeTask(taskId).build())).build(); } private class Execution { private final PersistentActionRequest request; - private final PersistentTask task; + private final NodePersistentTask task; private final PersistentActionRegistry.PersistentActionHolder holder; private final ActionListener listener; - Execution(PersistentActionRequest request, PersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, - ActionListener listener) { + Execution(PersistentActionRequest request, NodePersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, + ActionListener listener) { this.request = request; this.task = task; this.holder = holder; @@ -347,7 +347,7 @@ private class MockExecutor extends PersistentActionExecutor { } @Override - public void executeAction(Request request, PersistentTask task, + public void executeAction(Request request, NodePersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, ActionListener listener) { executions.add(new Execution(request, task, holder, listener)); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java index 6c9e7e7e3e7f0..4da92332dd14e 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java @@ -17,7 +17,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; @@ -68,8 +68,8 @@ public void testFullClusterRestart() throws Exception { } } final int numberOfRunningTasks = runningTasks; - PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasks.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); if (numberOfRunningTasks > 0) { @@ -85,11 +85,11 @@ public void testFullClusterRestart() throws Exception { internalCluster().fullRestart(); ensureYellow(); - tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksInProgress.TYPE); + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); // Check that cluster state is correct for (int i = 0; i < numberOfTasks; i++) { - PersistentTaskInProgress task = tasksInProgress.getTask(taskIds[i]); + PersistentTask task = tasksInProgress.getTask(taskIds[i]); assertNotNull(task); assertThat(task.isStopped(), equalTo(stopped[i])); } @@ -102,9 +102,9 @@ public void testFullClusterRestart() throws Exception { }); // Start all other tasks - tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksInProgress.TYPE); + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); for (int i = 0; i < numberOfTasks; i++) { - PersistentTaskInProgress task = tasksInProgress.getTask(taskIds[i]); + PersistentTask task = tasksInProgress.getTask(taskIds[i]); assertNotNull(task); logger.info("checking task with id {} stopped {} node {}", task.getId(), task.isStopped(), task.getExecutorNode()); assertThat(task.isStopped(), equalTo(stopped[i])); @@ -128,8 +128,8 @@ public void testFullClusterRestart() throws Exception { assertBusy(() -> { // Make sure the task is removed from the cluster state - assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksInProgress.TYPE)).tasks(), empty()); + assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasks.TYPE)).tasks(), empty()); }); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java index ec288c15a1c1b..be8b241b7004f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java @@ -126,8 +126,8 @@ public void testPersistentActionCompletionWithoutRemoval() throws Exception { .setRemoveOnCompletion(false) .setStopped(stopped).get().getTaskId(); - PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasks.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped)); assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue()); @@ -164,8 +164,8 @@ public void testPersistentActionCompletionWithoutRemoval() throws Exception { assertBusy(() -> { // Wait for the task to be marked as stopped - PersistentTasksInProgress tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasks.TYPE); assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.getTask(taskId).isStopped(), equalTo(true)); assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); @@ -216,8 +216,8 @@ public void testPersistentActionStatusUpdate() throws Exception { TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") .get().getTasks().get(0); - PersistentTasksInProgress tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasks.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue()); @@ -230,8 +230,8 @@ public void testPersistentActionStatusUpdate() throws Exception { int finalI = i; assertBusy(() -> { - PersistentTasksInProgress tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasks.TYPE); assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue()); assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); @@ -272,8 +272,8 @@ private void assertNoRunningTasks() throws Exception { assertThat(tasks.size(), equalTo(0)); // Make sure the task is removed from the cluster state - assertThat(((PersistentTasksInProgress) internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksInProgress.TYPE)).tasks(), empty()); + assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasks.TYPE)).tasks(), empty()); }); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java index 32f43783bb4af..466de5791089b 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java @@ -28,8 +28,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.PersistentTasks.Assignment; +import org.elasticsearch.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; import java.util.ArrayList; @@ -76,14 +76,14 @@ public Assignment getAssignment( public void testReassignTasksWithNoTasks() { ClusterState clusterState = initialState(); - assertThat(reassign(clusterState).metaData().custom(PersistentTasksInProgress.TYPE), nullValue()); + assertThat(reassign(clusterState).metaData().custom(PersistentTasks.TYPE), nullValue()); } public void testReassignConsidersClusterStateUpdates() { ClusterState clusterState = initialState(); ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder( - clusterState.metaData().custom(PersistentTasksInProgress.TYPE)); + PersistentTasks.Builder tasks = PersistentTasks.builder( + clusterState.metaData().custom(PersistentTasks.TYPE)); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(2, 40); @@ -91,11 +91,11 @@ public void testReassignConsidersClusterStateUpdates() { addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false); } - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksInProgress.TYPE, tasks.build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); clusterState = builder.metaData(metaData).nodes(nodes).build(); ClusterState newClusterState = reassign(clusterState); - PersistentTasksInProgress tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); assertThat(tasksInProgress, notNullValue()); } @@ -103,8 +103,8 @@ public void testReassignConsidersClusterStateUpdates() { public void testReassignTasks() { ClusterState clusterState = initialState(); ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder( - clusterState.metaData().custom(PersistentTasksInProgress.TYPE)); + PersistentTasks.Builder tasks = PersistentTasks.builder( + clusterState.metaData().custom(PersistentTasks.TYPE)); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(0, 40); @@ -128,11 +128,11 @@ public void testReassignTasks() { } } - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksInProgress.TYPE, tasks.build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); clusterState = builder.metaData(metaData).nodes(nodes).build(); ClusterState newClusterState = reassign(clusterState); - PersistentTasksInProgress tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); assertThat(tasksInProgress, notNullValue()); assertThat("number of tasks shouldn't change as a result or reassignment", @@ -140,7 +140,7 @@ public void testReassignTasks() { int assignOneCount = 0; - for (PersistentTaskInProgress task : tasksInProgress.tasks()) { + for (PersistentTask task : tasksInProgress.tasks()) { if (task.isStopped()) { assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue()); assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getAction())); @@ -151,7 +151,7 @@ public void testReassignTasks() { assertThat(task.getExecutorNode(), notNullValue()); assertThat(task.isAssigned(), equalTo(true)); if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { - logger.info(clusterState.metaData().custom(PersistentTasksInProgress.TYPE).toString()); + logger.info(clusterState.metaData().custom(PersistentTasks.TYPE).toString()); } assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); @@ -213,7 +213,7 @@ public Assignment getAssignment( private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { DiscoveryNodes nodes = clusterState.nodes(); - PersistentTasksInProgress tasksInProgress = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasksInProgress = clusterState.getMetaData().custom(PersistentTasks.TYPE); if (tasksInProgress.findTasks("assign_one", task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) { return randomNodeAssignment(clusterState.nodes()); @@ -242,15 +242,15 @@ private String dumpEvent(ClusterChangedEvent event) { return "nodes_changed: " + event.nodesChanged() + " nodes_removed:" + event.nodesRemoved() + " routing_table_changed:" + event.routingTableChanged() + - " tasks: " + event.state().metaData().custom(PersistentTasksInProgress.TYPE); + " tasks: " + event.state().metaData().custom(PersistentTasks.TYPE); } private ClusterState significantChange(ClusterState clusterState) { ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasksInProgress tasks = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); if (tasks != null) { if (randomBoolean()) { - for (PersistentTaskInProgress task : tasks.tasks()) { + for (PersistentTask task : tasks.tasks()) { if (task.isAssigned() && clusterState.nodes().nodeExists(task.getExecutorNode())) { logger.info("removed node {}", task.getExecutorNode()); builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(task.getExecutorNode())); @@ -265,11 +265,11 @@ private ClusterState significantChange(ClusterState clusterState) { // we don't have any unassigned tasks - add some if (randomBoolean()) { logger.info("added random task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), null, false); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), null, false); tasksOrNodesChanged = true; } else { logger.info("added unassignable task with custom assignment message"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksInProgress.builder(tasks), + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), new Assignment(null, "change me"), "never_assign", false); tasksOrNodesChanged = true; } @@ -292,11 +292,11 @@ private ClusterState significantChange(ClusterState clusterState) { return builder.build(); } - private PersistentTasksInProgress removeTasksWithChangingAssignment(PersistentTasksInProgress tasks) { + private PersistentTasks removeTasksWithChangingAssignment(PersistentTasks tasks) { if (tasks != null) { boolean changed = false; - PersistentTasksInProgress.Builder tasksBuilder = PersistentTasksInProgress.builder(tasks); - for (PersistentTaskInProgress task : tasks.tasks()) { + PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); + for (PersistentTask task : tasks.tasks()) { // Remove all unassigned tasks that cause changing assignments they might trigger a significant change if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) && "change me".equals(task.getAssignment().getExplanation())) { @@ -314,9 +314,9 @@ private PersistentTasksInProgress removeTasksWithChangingAssignment(PersistentTa private ClusterState insignificantChange(ClusterState clusterState) { ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasksInProgress tasks = clusterState.getMetaData().custom(PersistentTasksInProgress.TYPE); + PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); tasks = removeTasksWithChangingAssignment(tasks); - PersistentTasksInProgress.Builder tasksBuilder = PersistentTasksInProgress.builder(tasks); + PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); if (randomBoolean()) { if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { @@ -338,7 +338,7 @@ private ClusterState insignificantChange(ClusterState clusterState) { } else { logger.info("changed routing table"); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); - metaData.putCustom(PersistentTasksInProgress.TYPE, tasksBuilder.build()); + metaData.putCustom(PersistentTasks.TYPE, tasksBuilder.build()); RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); changeRoutingTable(metaData, routingTable); builder.metaData(metaData).routingTable(routingTable.build()); @@ -360,18 +360,18 @@ private ClusterState insignificantChange(ClusterState clusterState) { // clear the task if (randomBoolean()) { logger.info("removed all tasks"); - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksInProgress.TYPE, - PersistentTasksInProgress.builder().build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, + PersistentTasks.builder().build()); return builder.metaData(metaData).build(); } else { logger.info("set task custom to null"); - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasksInProgress.TYPE); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasks.TYPE); return builder.metaData(metaData).build(); } } logger.info("removed all unassigned tasks and changed routing table"); if (tasks != null) { - for (PersistentTaskInProgress task : tasks.tasks()) { + for (PersistentTask task : tasks.tasks()) { if (task.getExecutorNode() == null || "never_assign".equals(((TestRequest) task.getRequest()).getTestParam())) { tasksBuilder.removeTask(task.getId()); } @@ -384,11 +384,11 @@ private ClusterState insignificantChange(ClusterState clusterState) { .numberOfReplicas(1) .build(); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false) - .putCustom(PersistentTasksInProgress.TYPE, tasksBuilder.build()); + .putCustom(PersistentTasks.TYPE, tasksBuilder.build()); return builder.metaData(metaData).build(); } - private boolean hasAssignableTasks(PersistentTasksInProgress tasks, DiscoveryNodes discoveryNodes) { + private boolean hasAssignableTasks(PersistentTasks tasks, DiscoveryNodes discoveryNodes) { if (tasks == null || tasks.tasks().isEmpty()) { return false; } @@ -403,26 +403,26 @@ private boolean hasAssignableTasks(PersistentTasksInProgress tasks, DiscoveryNod }); } - private boolean hasTasksAssignedTo(PersistentTasksInProgress tasks, String nodeId) { + private boolean hasTasksAssignedTo(PersistentTasks tasks, String nodeId) { return tasks != null && tasks.tasks().stream().anyMatch( task -> nodeId.equals(task.getExecutorNode())) == false; } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, - MetaData.Builder metaData, PersistentTasksInProgress.Builder tasks, + MetaData.Builder metaData, PersistentTasks.Builder tasks, String node, boolean stopped) { return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)), randomAsciiOfLength(10), stopped); } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, - MetaData.Builder metaData, PersistentTasksInProgress.Builder tasks, + MetaData.Builder metaData, PersistentTasks.Builder tasks, Assignment assignment, String param, boolean stopped) { - return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksInProgress.TYPE, + return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasks.TYPE, tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build())); } - private void addTask(PersistentTasksInProgress.Builder tasks, String action, String param, String node, boolean stopped) { + private void addTask(PersistentTasks.Builder tasks, String action, String param, String node, boolean stopped) { tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action)); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksTests.java similarity index 86% rename from server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksTests.java index 0f55fdc4bfedc..caeb2583c3afa 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksInProgressTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksTests.java @@ -34,9 +34,9 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractDiffableSerializationTestCase; -import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; -import org.elasticsearch.persistent.PersistentTasksInProgress.Builder; -import org.elasticsearch.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.persistent.PersistentTasks.Assignment; +import org.elasticsearch.persistent.PersistentTasks.Builder; +import org.elasticsearch.persistent.PersistentTasks.PersistentTask; import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; @@ -50,12 +50,12 @@ import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; import static org.elasticsearch.persistent.TransportPersistentAction.NO_NODE_FOUND; -public class PersistentTasksInProgressTests extends AbstractDiffableSerializationTestCase { +public class PersistentTasksTests extends AbstractDiffableSerializationTestCase { @Override - protected PersistentTasksInProgress createTestInstance() { + protected PersistentTasks createTestInstance() { int numberOfTasks = randomInt(10); - PersistentTasksInProgress.Builder tasks = PersistentTasksInProgress.builder(); + PersistentTasks.Builder tasks = PersistentTasks.builder(); for (int i = 0; i < numberOfTasks; i++) { boolean stopped = randomBoolean(); tasks.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), @@ -70,14 +70,14 @@ protected PersistentTasksInProgress createTestInstance() { @Override protected Writeable.Reader instanceReader() { - return PersistentTasksInProgress::new; + return PersistentTasks::new; } @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Arrays.asList( - new Entry(MetaData.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), - new Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom), + new Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), + new Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), new Entry(Task.Status.class, Status.NAME, Status::new) )); @@ -85,7 +85,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { @Override protected Custom makeTestChanges(Custom testInstance) { - PersistentTasksInProgress tasksInProgress = (PersistentTasksInProgress) testInstance; + PersistentTasks tasksInProgress = (PersistentTasks) testInstance; Builder builder = new Builder(); switch (randomInt(3)) { case 0: @@ -118,12 +118,12 @@ protected Custom makeTestChanges(Custom testInstance) { @Override protected Writeable.Reader> diffReader() { - return PersistentTasksInProgress::readDiffFrom; + return PersistentTasks::readDiffFrom; } @Override - protected PersistentTasksInProgress doParseInstance(XContentParser parser) throws IOException { - return PersistentTasksInProgress.fromXContent(parser); + protected PersistentTasks doParseInstance(XContentParser parser) throws IOException { + return PersistentTasks.fromXContent(parser); } @Override @@ -155,7 +155,7 @@ private Builder addRandomTask(Builder builder) { return builder; } - private long pickRandomTask(PersistentTasksInProgress testInstance) { + private long pickRandomTask(PersistentTasks testInstance) { return randomFrom(new ArrayList<>(testInstance.tasks())).getId(); } @@ -170,9 +170,9 @@ protected NamedXContentRegistry xContentRegistry() { @SuppressWarnings("unchecked") public void testSerializationContext() throws Exception { - PersistentTasksInProgress testInstance = createTestInstance(); + PersistentTasks testInstance = createTestInstance(); for (int i = 0; i < randomInt(10); i++) { - testInstance = (PersistentTasksInProgress) makeTestChanges(testInstance); + testInstance = (PersistentTasks) makeTestChanges(testInstance); } ToXContent.MapParams params = new ToXContent.MapParams( @@ -183,12 +183,12 @@ public void testSerializationContext() throws Exception { XContentBuilder shuffled = shuffleXContent(builder); XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes()); - PersistentTasksInProgress newInstance = doParseInstance(parser); + PersistentTasks newInstance = doParseInstance(parser); assertNotSame(newInstance, testInstance); assertEquals(testInstance.tasks().size(), newInstance.tasks().size()); - for (PersistentTaskInProgress testTask : testInstance.tasks()) { - PersistentTaskInProgress newTask = (PersistentTaskInProgress) newInstance.getTask(testTask.getId()); + for (PersistentTask testTask : testInstance.tasks()) { + PersistentTask newTask = (PersistentTask) newInstance.getTask(testTask.getId()); assertNotNull(newTask); // Things that should be serialized @@ -205,14 +205,14 @@ public void testSerializationContext() throws Exception { } public void testBuilder() { - PersistentTasksInProgress persistentTasksInProgress = null; + PersistentTasks persistentTasks = null; long lastKnownTask = -1; for (int i = 0; i < randomIntBetween(10, 100); i++) { final Builder builder; if (randomBoolean()) { builder = new Builder(); } else { - builder = new Builder(persistentTasksInProgress); + builder = new Builder(persistentTasks); } boolean changed = false; for (int j = 0; j < randomIntBetween(1, 10); j++) { @@ -233,7 +233,7 @@ public void testBuilder() { break; case 2: if (builder.hasTask(lastKnownTask)) { - PersistentTaskInProgress task = builder.build().getTask(lastKnownTask); + PersistentTask task = builder.build().getTask(lastKnownTask); if (randomBoolean()) { // Trying to reassign to the same node builder.assignTask(lastKnownTask, (s, request) -> task.getAssignment()); @@ -276,7 +276,7 @@ public void testBuilder() { } } assertEquals(changed, builder.isChanged()); - persistentTasksInProgress = builder.build(); + persistentTasks = builder.build(); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java index 2299bace1667f..2c97bea928de6 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java @@ -63,7 +63,7 @@ import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.persistent.PersistentTasksInProgress.Assignment; +import org.elasticsearch.persistent.PersistentTasks.Assignment; import java.io.IOException; import java.util.ArrayList; @@ -121,8 +121,8 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), new NamedWriteableRegistry.Entry(Task.Status.class, PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), - new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), - new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) ); } @@ -130,8 +130,8 @@ public List getNamedWriteables() { @Override public List getNamedXContent() { return Arrays.asList( - new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksInProgress.TYPE), - PersistentTasksInProgress::fromXContent), + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE), + PersistentTasks::fromXContent), new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), TestRequest::fromXContent), new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) @@ -382,7 +382,7 @@ public Assignment getAssignment(TestRequest request, ClusterState clusterState) } @Override - protected void nodeOperation(PersistentTask task, TestRequest request, ActionListener listener) { + protected void nodeOperation(NodePersistentTask task, TestRequest request, ActionListener listener) { logger.info("started node operation for the task {}", task); try { TestTask testTask = (TestTask) task; @@ -465,7 +465,7 @@ public TestTasksRequestBuilder newRequestBuilder(ElasticsearchClient client) { } - public static class TestTask extends PersistentTask { + public static class TestTask extends NodePersistentTask { private volatile String operation; public TestTask(long id, String type, String action, String description, TaskId parentTask) { From 9bd24418d52a2299fe21460d24e68f655b2e7434 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Mar 2017 14:02:30 -0400 Subject: [PATCH 12/50] Make PersistentAction independent from TransportActions (#742) Removes the transport layer dependency from PersistentActions, makes PersistentActionRegistry immutable and rename actions into tasks in class and variable names. --- .../CompletionPersistentTaskAction.java | 38 +++- .../CreatePersistentTaskAction.java | 50 ++--- ....java => NodePersistentTasksExecutor.java} | 18 +- .../persistent/PersistentActionRegistry.java | 104 --------- .../persistent/PersistentActionService.java | 103 --------- ...equest.java => PersistentTaskRequest.java} | 4 +- ...ponse.java => PersistentTaskResponse.java} | 10 +- ...ava => PersistentTasksClusterService.java} | 64 +++--- ...ava => PersistentTasksCustomMetaData.java} | 107 ++++----- ...tion.java => PersistentTasksExecutor.java} | 56 ++--- .../PersistentTasksExecutorRegistry.java | 54 +++++ ...r.java => PersistentTasksNodeService.java} | 78 ++++--- .../persistent/PersistentTasksService.java | 148 +++++++++++++ .../RemovePersistentTaskAction.java | 8 +- .../persistent/StartPersistentTaskAction.java | 10 +- .../UpdatePersistentTaskStatusAction.java | 8 +- .../persistent/package-info.java | 31 ++- .../PersistentActionRegistryTests.java | 64 ------ ...> PersistentTasksClusterServiceTests.java} | 89 ++++---- ...> PersistentTasksCustomMetaDataTests.java} | 54 ++--- ...PersistentTasksExecutorFullRestartIT.java} | 68 +++--- ...IT.java => PersistentTasksExecutorIT.java} | 143 +++++++----- ...PersistentTasksExecutorResponseTests.java} | 10 +- ...ersistentTasksNodeServiceStatusTests.java} | 6 +- ...a => PersistentTasksNodeServiceTests.java} | 206 +++++++++--------- .../StartPersistentActionRequestTests.java | 6 +- ...in.java => TestPersistentTasksPlugin.java} | 106 +++------ .../UpdatePersistentTaskRequestTests.java | 2 +- 28 files changed, 820 insertions(+), 825 deletions(-) rename server/src/main/java/org/elasticsearch/persistent/{PersistentActionExecutor.java => NodePersistentTasksExecutor.java} (75%) delete mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java delete mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java rename server/src/main/java/org/elasticsearch/persistent/{PersistentActionRequest.java => PersistentTaskRequest.java} (88%) rename server/src/main/java/org/elasticsearch/persistent/{PersistentActionResponse.java => PersistentTaskResponse.java} (86%) rename server/src/main/java/org/elasticsearch/persistent/{PersistentTaskClusterService.java => PersistentTasksClusterService.java} (81%) rename server/src/main/java/org/elasticsearch/persistent/{PersistentTasks.java => PersistentTasksCustomMetaData.java} (85%) rename server/src/main/java/org/elasticsearch/persistent/{TransportPersistentAction.java => PersistentTasksExecutor.java} (65%) create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java rename server/src/main/java/org/elasticsearch/persistent/{PersistentActionCoordinator.java => PersistentTasksNodeService.java} (80%) create mode 100644 server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java delete mode 100644 server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java rename server/src/test/java/org/elasticsearch/persistent/{PersistentTaskClusterServiceTests.java => PersistentTasksClusterServiceTests.java} (84%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentTasksTests.java => PersistentTasksCustomMetaDataTests.java} (82%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentActionFullRestartIT.java => PersistentTasksExecutorFullRestartIT.java} (62%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentActionIT.java => PersistentTasksExecutorIT.java} (62%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentActionResponseTests.java => PersistentTasksExecutorResponseTests.java} (73%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentActionCoordinatorStatusTests.java => PersistentTasksNodeServiceStatusTests.java} (82%) rename server/src/test/java/org/elasticsearch/persistent/{PersistentActionCoordinatorTests.java => PersistentTasksNodeServiceTests.java} (59%) rename server/src/test/java/org/elasticsearch/persistent/{TestPersistentActionPlugin.java => TestPersistentTasksPlugin.java} (82%) diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java index ec31c7247037b..6a76774895003 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -118,6 +118,36 @@ public int hashCode() { } public static class Response extends AcknowledgedResponse { + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeAcknowledged(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AcknowledgedResponse that = (AcknowledgedResponse) o; + return isAcknowledged() == that.isAcknowledged(); + } + + @Override + public int hashCode() { + return Objects.hash(isAcknowledged()); + } } @@ -131,16 +161,16 @@ protected RequestBuilder(ElasticsearchClient client, CompletionPersistentTaskAct public static class TransportAction extends TransportMasterNodeAction { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -161,7 +191,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener() { + persistentTasksClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(newResponse()); diff --git a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java index d6a56ae928ce4..d0d52ed4a7030 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java @@ -43,7 +43,7 @@ * This action can be used to add the record for the persistent action to the cluster state. */ public class CreatePersistentTaskAction extends Action { public static final CreatePersistentTaskAction INSTANCE = new CreatePersistentTaskAction(); @@ -59,15 +59,15 @@ public RequestBuilder newRequestBuilder(ElasticsearchClient client) { } @Override - public PersistentActionResponse newResponse() { - return new PersistentActionResponse(); + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } public static class Request extends MasterNodeRequest { private String action; - private PersistentActionRequest request; + private PersistentTaskRequest request; private boolean stopped; @@ -77,7 +77,7 @@ public Request() { } - public Request(String action, PersistentActionRequest request) { + public Request(String action, PersistentTaskRequest request) { this.action = action; this.request = request; this.stopped = false; @@ -88,7 +88,7 @@ public Request(String action, PersistentActionRequest request) { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); action = in.readString(); - request = in.readNamedWriteable(PersistentActionRequest.class); + request = in.readNamedWriteable(PersistentTaskRequest.class); stopped = in.readBoolean(); removeOnCompletion = in.readBoolean(); } @@ -138,11 +138,11 @@ public void setAction(String action) { this.action = action; } - public PersistentActionRequest getRequest() { + public PersistentTaskRequest getRequest() { return request; } - public void setRequest(PersistentActionRequest request) { + public void setRequest(PersistentTaskRequest request) { this.request = request; } @@ -164,7 +164,7 @@ public void setRemoveOnCompletion(boolean removeOnCompletion) { } public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + PersistentTaskResponse, CreatePersistentTaskAction.RequestBuilder> { protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction action) { super(client, action, new Request()); @@ -175,8 +175,8 @@ public RequestBuilder setAction(String action) { return this; } - public RequestBuilder setRequest(PersistentActionRequest persistentActionRequest) { - request.setRequest(persistentActionRequest); + public RequestBuilder setRequest(PersistentTaskRequest persistentTaskRequest) { + request.setRequest(persistentTaskRequest); return this; } @@ -197,23 +197,23 @@ public RequestBuilder setRemoveOnCompletion(boolean removeOnCompletion) { } } - public static class TransportAction extends TransportMasterNodeAction { + public static class TransportAction extends TransportMasterNodeAction { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, - PersistentActionRegistry persistentActionRegistry, - PersistentActionService persistentActionService, + PersistentTasksClusterService persistentTasksClusterService, + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, + PersistentTasksService persistentTasksService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, CreatePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; - PersistentActionExecutor executor = new PersistentActionExecutor(threadPool); - clusterService.addListener(new PersistentActionCoordinator(settings, persistentActionService, persistentActionRegistry, - transportService.getTaskManager(), executor)); + this.persistentTasksClusterService = persistentTasksClusterService; + NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool); + clusterService.addListener(new PersistentTasksNodeService(settings, persistentTasksService, persistentTasksExecutorRegistry, + transportService.getTaskManager(), threadPool, executor)); } @Override @@ -222,8 +222,8 @@ protected String executor() { } @Override - protected PersistentActionResponse newResponse() { - return new PersistentActionResponse(); + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } @Override @@ -234,12 +234,12 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, - final ActionListener listener) { - persistentTaskClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion, + final ActionListener listener) { + persistentTasksClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion, new ActionListener() { @Override public void onResponse(Long newTaskId) { - listener.onResponse(new PersistentActionResponse(newTaskId)); + listener.onResponse(new PersistentTaskResponse(newTaskId)); } @Override diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java similarity index 75% rename from server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java rename to server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index 418ce8a36e325..3ad0bc57572e2 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -24,20 +24,20 @@ import org.elasticsearch.transport.TransportResponse.Empty; /** - * This component is responsible for execution of persistent actions. + * This component is responsible for execution of persistent tasks. */ -public class PersistentActionExecutor { +public class NodePersistentTasksExecutor { private final ThreadPool threadPool; - public PersistentActionExecutor(ThreadPool threadPool) { + public NodePersistentTasksExecutor(ThreadPool threadPool) { this.threadPool = threadPool; } - public void executeAction(Request request, - NodePersistentTask task, - PersistentActionRegistry.PersistentActionHolder holder, - ActionListener listener) { - threadPool.executor(holder.getExecutor()).execute(new AbstractRunnable() { + public void executeTask(Request request, + NodePersistentTask task, + PersistentTasksExecutor action, + ActionListener listener) { + threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { listener.onFailure(e); @@ -47,7 +47,7 @@ public void onFailure(Exception e) { @Override protected void doRun() throws Exception { try { - holder.getPersistentAction().nodeOperation(task, request, listener); + action.nodeOperation(task, request, listener); } catch (Exception ex) { listener.onFailure(ex); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java deleted file mode 100644 index c0e9697a284f1..0000000000000 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRegistry.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.persistent; - -import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.settings.Settings; - -import java.util.Collections; -import java.util.Map; - -/** - * Components that registers all persistent actions - */ -public class PersistentActionRegistry extends AbstractComponent { - - private volatile Map> actions = Collections.emptyMap(); - - private final Object actionHandlerMutex = new Object(); - - public PersistentActionRegistry(Settings settings) { - super(settings); - } - - public void registerPersistentAction(String action, - TransportPersistentAction persistentAction) { - registerPersistentAction(new PersistentActionHolder<>(action, persistentAction, persistentAction.getExecutor())); - } - - private void registerPersistentAction( - PersistentActionHolder reg) { - - synchronized (actionHandlerMutex) { - PersistentActionHolder replaced = actions.get(reg.getAction()); - actions = MapBuilder.newMapBuilder(actions).put(reg.getAction(), reg).immutableMap(); - if (replaced != null) { - logger.warn("registered two handlers for persistent action {}, handlers: {}, {}", reg.getAction(), reg, replaced); - } - } - } - - public void removeHandler(String action) { - synchronized (actionHandlerMutex) { - actions = MapBuilder.newMapBuilder(actions).remove(action).immutableMap(); - } - } - - @SuppressWarnings("unchecked") - public PersistentActionHolder getPersistentActionHolderSafe(String action) { - PersistentActionHolder holder = (PersistentActionHolder) actions.get(action); - if (holder == null) { - throw new IllegalStateException("Unknown persistent action [" + action + "]"); - } - return holder; - } - - public - TransportPersistentAction getPersistentActionSafe(String action) { - PersistentActionHolder holder = getPersistentActionHolderSafe(action); - return holder.getPersistentAction(); - } - - public static final class PersistentActionHolder { - - private final String action; - private final TransportPersistentAction persistentAction; - private final String executor; - - - public PersistentActionHolder(String action, TransportPersistentAction persistentAction, String executor) { - this.action = action; - this.persistentAction = persistentAction; - this.executor = executor; - } - - public String getAction() { - return action; - } - - public TransportPersistentAction getPersistentAction() { - return persistentAction; - } - - public String getExecutor() { - return executor; - } - } -} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java deleted file mode 100644 index 4cd553b84c7a5..0000000000000 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionService.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.persistent; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; - -/** - * Service responsible for executing restartable actions that can survive disappearance of a coordinating and executor nodes. - */ -public class PersistentActionService extends AbstractComponent { - - private final Client client; - private final ThreadPool threadPool; - private final ClusterService clusterService; - - public PersistentActionService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client) { - super(settings); - this.client = client; - this.threadPool = threadPool; - this.clusterService = clusterService; - } - - public void sendRequest(String action, Request request, - ActionListener listener) { - CreatePersistentTaskAction.Request startRequest = new CreatePersistentTaskAction.Request(action, request); - try { - client.execute(CreatePersistentTaskAction.INSTANCE, startRequest, listener); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public void sendCompletionNotification(long taskId, Exception failure, - ActionListener listener) { - CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); - // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier. - // reason [the applied cluster state is not yet available]) - try { - threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - - @Override - protected void doRun() throws Exception { - client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener); - } - }); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public void sendCancellation(long taskId, ActionListener listener) { - DiscoveryNode localNode = clusterService.localNode(); - CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); - cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); - cancelTasksRequest.setReason("persistent action was removed"); - try { - client.admin().cluster().cancelTasks(cancelTasksRequest, listener); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public void updateStatus(long taskId, Task.Status status, ActionListener listener) { - UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status); - try { - client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, listener); - } catch (Exception e) { - listener.onFailure(e); - } - } - -} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java similarity index 88% rename from server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java index dfd1da1ea438d..b4c192ceab4c6 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionRequest.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java @@ -25,9 +25,9 @@ import org.elasticsearch.tasks.TaskId; /** - * Base class for a request for a persistent action + * Base class for a request for a persistent task */ -public abstract class PersistentActionRequest extends ActionRequest implements NamedWriteable, ToXContent { +public abstract class PersistentTaskRequest extends ActionRequest implements NamedWriteable, ToXContent { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId) { return new NodePersistentTask(id, type, action, getDescription(), parentTaskId); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionResponse.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java similarity index 86% rename from server/src/main/java/org/elasticsearch/persistent/PersistentActionResponse.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java index 69362198975cb..14ce2e3036522 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionResponse.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java @@ -26,16 +26,16 @@ import java.util.Objects; /** - * Response upon a successful start or an persistent action + * Response upon a successful start or an persistent task */ -public class PersistentActionResponse extends ActionResponse { +public class PersistentTaskResponse extends ActionResponse { private long taskId; - public PersistentActionResponse() { + public PersistentTaskResponse() { super(); } - public PersistentActionResponse(long taskId) { + public PersistentTaskResponse(long taskId) { this.taskId = taskId; } @@ -59,7 +59,7 @@ public long getTaskId() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - PersistentActionResponse that = (PersistentActionResponse) o; + PersistentTaskResponse that = (PersistentTaskResponse) o; return taskId == that.taskId; } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java similarity index 81% rename from server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 372fd05ed1e14..a66acc2bbbf81 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -33,20 +33,20 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.persistent.PersistentTasks.Assignment; -import org.elasticsearch.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.util.Objects; /** * Component that runs only on the master node and is responsible for assigning running tasks to nodes */ -public class PersistentTaskClusterService extends AbstractComponent implements ClusterStateListener { +public class PersistentTasksClusterService extends AbstractComponent implements ClusterStateListener { private final ClusterService clusterService; - private final PersistentActionRegistry registry; + private final PersistentTasksExecutorRegistry registry; - public PersistentTaskClusterService(Settings settings, PersistentActionRegistry registry, ClusterService clusterService) { + public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorRegistry registry, ClusterService clusterService) { super(settings); this.clusterService = clusterService; clusterService.addListener(this); @@ -61,15 +61,17 @@ public PersistentTaskClusterService(Settings settings, PersistentActionRegistry * @param request request * @param listener the listener that will be called when task is started */ - public void createPersistentTask(String action, Request request, boolean stopped, - boolean removeOnCompletion, - ActionListener listener) { + public void createPersistentTask(String action, Request request, boolean stopped, + boolean removeOnCompletion, + ActionListener listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { + validate(action, clusterService.state(), request); final Assignment assignment; if (stopped) { - assignment = PersistentTasks.FINISHED_TASK_ASSIGNMENT; // the task is stopped no need to assign it anywhere + // the task is stopped no need to assign it anywhere + assignment = PersistentTasksCustomMetaData.FINISHED_TASK_ASSIGNMENT; } else { assignment = getAssignement(action, currentState, request); } @@ -84,7 +86,7 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { listener.onResponse( - ((PersistentTasks) newState.getMetaData().custom(PersistentTasks.TYPE)).getCurrentId()); + ((PersistentTasksCustomMetaData) newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)).getCurrentId()); } }); } @@ -108,7 +110,7 @@ public void completeOrRestartPersistentTask(long id, Exception failure, ActionLi clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { if (failure != null) { // If the task failed - we need to restart it on another node, otherwise we just remove it @@ -146,7 +148,7 @@ public void startPersistentTask(long id, ActionListener listener) { clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress .assignTask(id, (action, request) -> getAssignement(action, currentState, request))); @@ -177,7 +179,7 @@ public void removePersistentTask(long id, ActionListener listener) { clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress.removeTask(id)); } else { @@ -208,7 +210,7 @@ public void updatePersistentTaskStatus(long id, Task.Status status, ActionListen clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasks.Builder tasksInProgress = builder(currentState); + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { return update(currentState, tasksInProgress.updateTaskStatus(id, status)); } else { @@ -228,10 +230,14 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } - private Assignment getAssignement(String action, ClusterState currentState, Request request) { - TransportPersistentAction persistentAction = registry.getPersistentActionSafe(action); - persistentAction.validate(request, currentState); - return persistentAction.getAssignment(request, currentState); + private Assignment getAssignement(String taskName, ClusterState currentState, Request request) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + return persistentTasksExecutor.getAssignment(request, currentState); + } + + private void validate(String taskName, ClusterState currentState, Request request) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + persistentTasksExecutor.validate(request, currentState); } @Override @@ -248,12 +254,12 @@ public void clusterChanged(ClusterChangedEvent event) { } interface ExecutorNodeDecider { - Assignment getAssignment(String action, ClusterState currentState, Request request); + Assignment getAssignment(String action, ClusterState currentState, Request request); } static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { - PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); - PersistentTasks prevTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData prevTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks != null && (Objects.equals(tasks, prevTasks) == false || event.nodesChanged() || event.routingTableChanged() || @@ -264,7 +270,7 @@ static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecid if (taskInProgress.needsReassignment(event.state().nodes())) { // there is an unassigned task or task with a disappeared node - we need to try assigning it if (Objects.equals(taskInProgress.getAssignment(), - decider.getAssignment(taskInProgress.getAction(), event.state(), taskInProgress.getRequest())) == false) { + decider.getAssignment(taskInProgress.getTaskName(), event.state(), taskInProgress.getRequest())) == false) { // it looks like a assignment for at least one task is possible - let's trigger reassignment reassignmentRequired = true; break; @@ -284,7 +290,7 @@ public void reassignTasks() { clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { - return reassignTasks(currentState, logger, PersistentTaskClusterService.this::getAssignement); + return reassignTasks(currentState, logger, PersistentTasksClusterService.this::getAssignement); } @Override @@ -300,7 +306,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) { - PersistentTasks tasks = currentState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); ClusterState clusterState = currentState; DiscoveryNodes nodes = currentState.nodes(); if (tasks != null) { @@ -309,7 +315,7 @@ static ClusterState reassignTasks(ClusterState currentState, Logger logger, Exec for (PersistentTask task : tasks.tasks()) { if (task.needsReassignment(nodes)) { // there is an unassigned task - we need to try assigning it - Assignment assignment = decider.getAssignment(task.getAction(), clusterState, task.getRequest()); + Assignment assignment = decider.getAssignment(task.getTaskName(), clusterState, task.getRequest()); if (Objects.equals(assignment, task.getAssignment()) == false) { logger.trace("reassigning task {} from node {} to node {}", task.getId(), task.getAssignment().getExecutorNode(), assignment.getExecutorNode()); @@ -329,14 +335,14 @@ static ClusterState reassignTasks(ClusterState currentState, Logger logger, Exec return clusterState; } - private static PersistentTasks.Builder builder(ClusterState currentState) { - return PersistentTasks.builder(currentState.getMetaData().custom(PersistentTasks.TYPE)); + private static PersistentTasksCustomMetaData.Builder builder(ClusterState currentState) { + return PersistentTasksCustomMetaData.builder(currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); } - private static ClusterState update(ClusterState currentState, PersistentTasks.Builder tasksInProgress) { + private static ClusterState update(ClusterState currentState, PersistentTasksCustomMetaData.Builder tasksInProgress) { if (tasksInProgress.isChanged()) { return ClusterState.builder(currentState).metaData( - MetaData.builder(currentState.metaData()).putCustom(PersistentTasks.TYPE, tasksInProgress.build()) + MetaData.builder(currentState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasksInProgress.build()) ).build(); } else { return currentState; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasks.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java similarity index 85% rename from server/src/main/java/org/elasticsearch/persistent/PersistentTasks.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 855178c760e26..c78cb07e8c764 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasks.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -56,7 +56,7 @@ /** * A cluster state record that contains a list of all running persistent tasks */ -public final class PersistentTasks extends AbstractNamedDiffable implements MetaData.Custom { +public final class PersistentTasksCustomMetaData extends AbstractNamedDiffable implements MetaData.Custom { public static final String TYPE = "persistent_tasks"; private static final String API_CONTEXT = MetaData.XContentContext.API.toString(); @@ -66,21 +66,21 @@ public final class PersistentTasks extends AbstractNamedDiffable> tasks) { + public PersistentTasksCustomMetaData(long currentId, Map> tasks) { this.currentId = currentId; this.tasks = tasks; } private static final ObjectParser PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new); - private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = + private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = new ObjectParser<>("tasks", TaskBuilder::new); public static final ConstructingObjectParser ASSIGNMENT_PARSER = new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); - private static final NamedObjectParser REQUEST_PARSER = - (XContentParser p, Void c, String name) -> p.namedObject(PersistentActionRequest.class, name, null); + private static final NamedObjectParser REQUEST_PARSER = + (XContentParser p, Void c, String name) -> p.namedObject(PersistentTaskRequest.class, name, null); private static final NamedObjectParser STATUS_PARSER = (XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null); @@ -95,20 +95,20 @@ public PersistentTasks(long currentId, Map> tasks) { // Task parser initialization PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setId, new ParseField("id")); - PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setAction, new ParseField("action")); + PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setTaskName, new ParseField("name")); PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id")); PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setRemoveOnCompletion, new ParseField("remove_on_completion")); PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setStopped, new ParseField("stopped")); PERSISTENT_TASK_PARSER.declareNamedObjects( - (TaskBuilder taskBuilder, List objects) -> { + (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { - throw new IllegalArgumentException("only one action request per task is allowed"); + throw new IllegalArgumentException("only one request per task is allowed"); } taskBuilder.setRequest(objects.get(0)); }, REQUEST_PARSER, new ParseField("request")); PERSISTENT_TASK_PARSER.declareNamedObjects( - (TaskBuilder taskBuilder, List objects) -> { + (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { throw new IllegalArgumentException("only one status per task is allowed"); } @@ -133,16 +133,16 @@ public PersistentTask getTask(long id) { return this.tasks.get(id); } - public Collection> findTasks(String actionName, Predicate> predicate) { + public Collection> findTasks(String taskName, Predicate> predicate) { return this.tasks().stream() - .filter(p -> actionName.equals(p.getAction())) + .filter(p -> taskName.equals(p.getTaskName())) .filter(predicate) .collect(Collectors.toList()); } - public boolean tasksExist(String actionName, Predicate> predicate) { + public boolean tasksExist(String taskName, Predicate> predicate) { return this.tasks().stream() - .filter(p -> actionName.equals(p.getAction())) + .filter(p -> taskName.equals(p.getTaskName())) .anyMatch(predicate); } @@ -150,7 +150,7 @@ public boolean tasksExist(String actionName, Predicate> predic public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - PersistentTasks that = (PersistentTasks) o; + PersistentTasksCustomMetaData that = (PersistentTasksCustomMetaData) o; return currentId == that.currentId && Objects.equals(tasks, that.tasks); } @@ -165,8 +165,9 @@ public String toString() { return Strings.toString(this); } - public long getNumberOfTasksOnNode(String nodeId, String action) { - return tasks.values().stream().filter(task -> action.equals(task.action) && nodeId.equals(task.assignment.executorNode)).count(); + public long getNumberOfTasksOnNode(String nodeId, String taskName) { + return tasks.values().stream().filter( + task -> taskName.equals(task.taskName) && nodeId.equals(task.assignment.executorNode)).count(); } @Override @@ -179,7 +180,7 @@ public EnumSet context() { return ALL_CONTEXTS; } - public static PersistentTasks fromXContent(XContentParser parser) throws IOException { + public static PersistentTasksCustomMetaData fromXContent(XContentParser parser) throws IOException { return PERSISTENT_TASKS_PARSER.parse(parser, null).build(); } @@ -234,10 +235,10 @@ public String toString() { /** * A record that represents a single running persistent task */ - public static class PersistentTask implements Writeable, ToXContent { + public static class PersistentTask implements Writeable, ToXContent { private final long id; private final long allocationId; - private final String action; + private final String taskName; private final Request request; private final boolean stopped; private final boolean removeOnCompletion; @@ -248,25 +249,25 @@ public static class PersistentTask impl private final Long allocationIdOnLastStatusUpdate; - public PersistentTask(long id, String action, Request request, boolean stopped, boolean removeOnCompletion, Assignment assignment) { - this(id, 0L, action, request, stopped, removeOnCompletion, null, assignment, null); + public PersistentTask(long id, String taskName, Request request, boolean stopped, boolean removeOnCompletion, Assignment assignment) { + this(id, 0L, taskName, request, stopped, removeOnCompletion, null, assignment, null); } public PersistentTask(PersistentTask task, boolean stopped, Assignment assignment) { - this(task.id, task.allocationId + 1L, task.action, task.request, stopped, task.removeOnCompletion, task.status, + this(task.id, task.allocationId + 1L, task.taskName, task.request, stopped, task.removeOnCompletion, task.status, assignment, task.allocationId); } public PersistentTask(PersistentTask task, Status status) { - this(task.id, task.allocationId, task.action, task.request, task.stopped, task.removeOnCompletion, status, + this(task.id, task.allocationId, task.taskName, task.request, task.stopped, task.removeOnCompletion, status, task.assignment, task.allocationId); } - private PersistentTask(long id, long allocationId, String action, Request request, boolean stopped, boolean removeOnCompletion, + private PersistentTask(long id, long allocationId, String taskName, Request request, boolean stopped, boolean removeOnCompletion, Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; - this.action = action; + this.taskName = taskName; this.request = request; this.status = status; this.stopped = stopped; @@ -281,8 +282,8 @@ private PersistentTask(long id, long allocationId, String action, Request reques private PersistentTask(StreamInput in) throws IOException { id = in.readLong(); allocationId = in.readLong(); - action = in.readString(); - request = (Request) in.readNamedWriteable(PersistentActionRequest.class); + taskName = in.readString(); + request = (Request) in.readNamedWriteable(PersistentTaskRequest.class); stopped = in.readBoolean(); removeOnCompletion = in.readBoolean(); status = in.readOptionalNamedWriteable(Task.Status.class); @@ -294,7 +295,7 @@ private PersistentTask(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeLong(id); out.writeLong(allocationId); - out.writeString(action); + out.writeString(taskName); out.writeNamedWriteable(request); out.writeBoolean(stopped); out.writeBoolean(removeOnCompletion); @@ -311,7 +312,7 @@ public boolean equals(Object o) { PersistentTask that = (PersistentTask) o; return id == that.id && allocationId == that.allocationId && - Objects.equals(action, that.action) && + Objects.equals(taskName, that.taskName) && Objects.equals(request, that.request) && stopped == that.stopped && removeOnCompletion == that.removeOnCompletion && @@ -322,7 +323,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(id, allocationId, action, request, stopped, removeOnCompletion, status, assignment, + return Objects.hash(id, allocationId, taskName, request, stopped, removeOnCompletion, status, assignment, allocationIdOnLastStatusUpdate); } @@ -339,8 +340,8 @@ public long getAllocationId() { return allocationId; } - public String getAction() { - return action; + public String getTaskName() { + return taskName; } public Request getRequest() { @@ -393,7 +394,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field("id", id); - builder.field("action", action); + builder.field("name", taskName); builder.startObject("request"); { builder.field(request.getWriteableName(), request, params); @@ -433,10 +434,10 @@ public boolean isFragment() { } } - private static class TaskBuilder { + private static class TaskBuilder { private long id; private long allocationId; - private String action; + private String taskName; private Request request; private boolean stopped = true; private boolean removeOnCompletion; @@ -454,8 +455,8 @@ public TaskBuilder setAllocationId(long allocationId) { return this; } - public TaskBuilder setAction(String action) { - this.action = action; + public TaskBuilder setTaskName(String taskName) { + this.taskName = taskName; return this; } @@ -491,7 +492,7 @@ public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdO } public PersistentTask build() { - return new PersistentTask<>(id, allocationId, action, request, stopped, removeOnCompletion, status, + return new PersistentTask<>(id, allocationId, taskName, request, stopped, removeOnCompletion, status, assignment, allocationIdOnLastStatusUpdate); } } @@ -501,7 +502,7 @@ public String getWriteableName() { return TYPE; } - public PersistentTasks(StreamInput in) throws IOException { + public PersistentTasksCustomMetaData(StreamInput in) throws IOException { currentId = in.readLong(); tasks = in.readMap(StreamInput::readLong, PersistentTask::new); } @@ -538,7 +539,7 @@ public static Builder builder() { return new Builder(); } - public static Builder builder(PersistentTasks tasks) { + public static Builder builder(PersistentTasksCustomMetaData tasks) { return new Builder(tasks); } @@ -550,7 +551,7 @@ public static class Builder { public Builder() { } - public Builder(PersistentTasks tasksInProgress) { + public Builder(PersistentTasksCustomMetaData tasksInProgress) { if (tasksInProgress != null) { tasks.putAll(tasksInProgress.tasks); currentId = tasksInProgress.currentId; @@ -564,7 +565,7 @@ private Builder setCurrentId(long currentId) { return this; } - private Builder setTasks(List> tasks) { + private Builder setTasks(List> tasks) { for (TaskBuilder builder : tasks) { PersistentTask task = builder.build(); this.tasks.put(task.getId(), task); @@ -577,11 +578,11 @@ private Builder setTasks(List * After the task is added its id can be found by calling {{@link #getCurrentId()}} method. */ - public Builder addTask(String action, Request request, boolean stopped, - boolean removeOnCompletion, Assignment assignment) { + public Builder addTask(String taskName, Request request, boolean stopped, + boolean removeOnCompletion, Assignment assignment) { changed = true; currentId++; - tasks.put(currentId, new PersistentTask<>(currentId, action, request, stopped, removeOnCompletion, assignment)); + tasks.put(currentId, new PersistentTask<>(currentId, taskName, request, stopped, removeOnCompletion, assignment)); return this; } @@ -604,11 +605,11 @@ public Builder reassignTask(long taskId, Assignment assignment) { * {@link #reassignTask(long, BiFunction)} instead */ @SuppressWarnings("unchecked") - public Builder assignTask(long taskId, - BiFunction executorNodeFunc) { + public Builder assignTask(long taskId, + BiFunction executorNodeFunc) { PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks - Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request); if (assignment.isAssigned() || taskInProgress.isStopped()) { changed = true; tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); @@ -621,12 +622,12 @@ public Builder assignTask(long taskId, * Reassigns the task to another node if the task exist */ @SuppressWarnings("unchecked") - public Builder reassignTask(long taskId, - BiFunction executorNodeFunc) { + public Builder reassignTask(long taskId, + BiFunction executorNodeFunc) { PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); if (taskInProgress != null) { changed = true; - Assignment assignment = executorNodeFunc.apply(taskInProgress.action, taskInProgress.request); + Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request); tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); } return this; @@ -693,8 +694,8 @@ public boolean isChanged() { return changed; } - public PersistentTasks build() { - return new PersistentTasks(currentId, Collections.unmodifiableMap(tasks)); + public PersistentTasksCustomMetaData build() { + return new PersistentTasksCustomMetaData(currentId, Collections.unmodifiableMap(tasks)); } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java similarity index 65% rename from server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index a4c36884108af..862bd15c56d9b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/TransportPersistentAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -20,41 +20,37 @@ package org.elasticsearch.persistent; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.transport.TransportService; -import org.elasticsearch.persistent.PersistentTasks.Assignment; +import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import java.util.function.Predicate; -import java.util.function.Supplier; /** - * An action that can survive restart of requesting or executing node. - * These actions are using cluster state rather than only transport service to send requests and responses. + * An executor of tasks that can survive restart of requesting or executing node. + * These tasks are using cluster state rather than only transport service to send requests and responses. */ -public abstract class TransportPersistentAction - extends HandledTransportAction { +public abstract class PersistentTasksExecutor extends AbstractComponent { private final String executor; - private final PersistentActionService persistentActionService; - - protected TransportPersistentAction(Settings settings, String actionName, boolean canTripCircuitBreaker, ThreadPool threadPool, - TransportService transportService, PersistentActionService persistentActionService, - PersistentActionRegistry persistentActionRegistry, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier requestSupplier, String executor) { - super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver, - requestSupplier); + private final String taskName; + private final PersistentTasksService persistentTasksService; + + protected PersistentTasksExecutor(Settings settings, String taskName, PersistentTasksService persistentTasksService, + String executor) { + super(settings); + this.taskName = taskName; this.executor = executor; - this.persistentActionService = persistentActionService; - persistentActionRegistry.registerPersistentAction(actionName, this); + this.persistentTasksService = persistentTasksService; + } + + public String getTaskName() { + return taskName; } public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment"); @@ -73,21 +69,20 @@ public Assignment getAssignment(Request request, ClusterState clusterState) { } } - /** * Finds the least loaded node that satisfies the selector criteria */ protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate selector) { long minLoad = Long.MAX_VALUE; DiscoveryNode minLoadedNode = null; - PersistentTasks persistentTasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); for (DiscoveryNode node : clusterState.getNodes()) { if (selector.test(node)) { if (persistentTasks == null) { // We don't have any task running yet, pick the first available node return node; } - long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), actionName); + long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), taskName); if (minLoad > numberOfTasks) { minLoad = numberOfTasks; minLoadedNode = node; @@ -106,11 +101,6 @@ public void validate(Request request, ClusterState clusterState) { } - @Override - protected void doExecute(Request request, ActionListener listener) { - persistentActionService.sendRequest(actionName, request, listener); - } - /** * Updates the persistent task status in the cluster state. *

@@ -118,10 +108,10 @@ protected void doExecute(Request request, ActionListener listener) { - persistentActionService.updateStatus(task.getPersistentTaskId(), status, - new ActionListener() { + persistentTasksService.updateStatus(task.getPersistentTaskId(), status, + new PersistentTaskOperationListener() { @Override - public void onResponse(UpdatePersistentTaskStatusAction.Response response) { + public void onResponse(long taskId) { listener.onResponse(Empty.INSTANCE); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java new file mode 100644 index 0000000000000..e506f0ad748e3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Components that registers all persistent task executors + */ +public class PersistentTasksExecutorRegistry extends AbstractComponent { + + private final Map> taskExecutors; + + @SuppressWarnings("unchecked") + public PersistentTasksExecutorRegistry(Settings settings, Collection> taskExecutors) { + super(settings); + Map> map = new HashMap<>(); + for (PersistentTasksExecutor executor : taskExecutors) { + map.put(executor.getTaskName(), executor); + } + this.taskExecutors = Collections.unmodifiableMap(map); + } + + @SuppressWarnings("unchecked") + public PersistentTasksExecutor getPersistentTaskExecutorSafe(String taskName) { + PersistentTasksExecutor executor = (PersistentTasksExecutor) taskExecutors.get(taskName); + if (executor == null) { + throw new IllegalStateException("Unknown persistent executor [" + taskName + "]"); + } + return executor; + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java similarity index 80% rename from server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index 427fa129f89f1..22fdca1b869ca 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentActionCoordinator.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -22,7 +22,6 @@ import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.common.Nullable; @@ -32,12 +31,15 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.persistent.PersistentTasks.PersistentTask; +import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.HashMap; @@ -50,33 +52,35 @@ import static java.util.Objects.requireNonNull; /** - * This component is responsible for coordination of execution of persistent actions on individual nodes. It runs on all + * This component is responsible for coordination of execution of persistent tasks on individual nodes. It runs on all * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. */ -public class PersistentActionCoordinator extends AbstractComponent implements ClusterStateListener { +public class PersistentTasksNodeService extends AbstractComponent implements ClusterStateListener { private final Map runningTasks = new HashMap<>(); - private final PersistentActionService persistentActionService; - private final PersistentActionRegistry persistentActionRegistry; + private final PersistentTasksService persistentTasksService; + private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry; private final TaskManager taskManager; - private final PersistentActionExecutor persistentActionExecutor; + private final ThreadPool threadPool; + private final NodePersistentTasksExecutor nodePersistentTasksExecutor; - public PersistentActionCoordinator(Settings settings, - PersistentActionService persistentActionService, - PersistentActionRegistry persistentActionRegistry, - TaskManager taskManager, - PersistentActionExecutor persistentActionExecutor) { + public PersistentTasksNodeService(Settings settings, + PersistentTasksService persistentTasksService, + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, + TaskManager taskManager, ThreadPool threadPool, + NodePersistentTasksExecutor nodePersistentTasksExecutor) { super(settings); - this.persistentActionService = persistentActionService; - this.persistentActionRegistry = persistentActionRegistry; + this.persistentTasksService = persistentTasksService; + this.persistentTasksExecutorRegistry = persistentTasksExecutorRegistry; this.taskManager = taskManager; - this.persistentActionExecutor = persistentActionExecutor; + this.threadPool = threadPool; + this.nodePersistentTasksExecutor = nodePersistentTasksExecutor; } @Override public void clusterChanged(ClusterChangedEvent event) { - PersistentTasks tasks = event.state().getMetaData().custom(PersistentTasks.TYPE); - PersistentTasks previousTasks = event.previousState().getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData previousTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { // We have some changes let's check if they are related to our node @@ -124,10 +128,9 @@ public void clusterChanged(ClusterChangedEvent event) { } - private void startTask(PersistentTask taskInProgress) { - PersistentActionRegistry.PersistentActionHolder holder = - persistentActionRegistry.getPersistentActionHolderSafe(taskInProgress.getAction()); - NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]", + private void startTask(PersistentTask taskInProgress) { + PersistentTasksExecutor action = persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName()); + NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]", taskInProgress.getRequest()); boolean processed = false; try { @@ -137,7 +140,7 @@ private void startTask(PersistentTask< PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask); try { runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask); - persistentActionExecutor.executeAction(taskInProgress.getRequest(), task, holder, listener); + nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action, listener); } catch (Exception e) { // Submit task failure listener.onFailure(e); @@ -162,9 +165,11 @@ private void cancelTask(PersistentTaskId persistentTaskId) { RunningPersistentTask task = runningTasks.remove(persistentTaskId); if (task != null && task.getTask() != null) { if (task.markAsCancelled()) { - persistentActionService.sendCancellation(task.getTask().getId(), new ActionListener() { + persistentTasksService.sendCancellation(task.getTask().getId(), new PersistentTaskOperationListener() { @Override - public void onResponse(CancelTasksResponse cancelTasksResponse) { + public void onResponse(long taskId) { + logger.trace("Persistent task with id {} was cancelled", taskId); + } @Override @@ -184,7 +189,24 @@ private void restartCompletionNotification(RunningPersistentTask task) { taskManager.unregister(task.getTask()); } else { if (task.restartCompletionNotification()) { - persistentActionService.sendCompletionNotification(task.getId(), task.getFailure(), new PublishedResponseListener(task)); + // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier. + // reason [the applied cluster state is not yet available]) + PublishedResponseListener listener = new PublishedResponseListener(task); + try { + threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @Override + protected void doRun() throws Exception { + persistentTasksService.sendCompletionNotification(task.getId(), task.getFailure(), listener); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } } else { logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState()); } @@ -197,7 +219,7 @@ private void startCompletionNotification(RunningPersistentTask task, Exception e } else { logger.trace("sending notification for failed task {}", task.getId()); if (task.startNotification(e)) { - persistentActionService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); + persistentTasksService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); } else { logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState()); } @@ -236,7 +258,7 @@ public void onFailure(Exception e) { } } - private class PublishedResponseListener implements ActionListener { + private class PublishedResponseListener implements PersistentTaskOperationListener { private final RunningPersistentTask task; PublishedResponseListener(final RunningPersistentTask task) { @@ -245,7 +267,7 @@ private class PublishedResponseListener implements ActionListener void createPersistentActionTask(String action, Request request, + PersistentTaskOperationListener listener) { + createPersistentActionTask(action, request, false, true, listener); + } + + /** + * Creates the specified persistent action. The action is started unless the stopped parameter is equal to true. + * If removeOnCompletion parameter is equal to true, the task is removed from the cluster state upon completion. + * Otherwise it will remain there in the stopped state. + */ + public void createPersistentActionTask(String action, Request request, + boolean stopped, + boolean removeOnCompletion, + PersistentTaskOperationListener listener) { + CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(action, request); + createPersistentActionRequest.setStopped(stopped); + createPersistentActionRequest.setRemoveOnCompletion(removeOnCompletion); + try { + client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( + o -> listener.onResponse(o.getTaskId()), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure + * + */ + public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) { + CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); + try { + client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Cancels the persistent task. + */ + public void sendCancellation(long taskId, PersistentTaskOperationListener listener) { + DiscoveryNode localNode = clusterService.localNode(); + CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); + cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); + cancelTasksRequest.setReason("persistent action was removed"); + try { + client.admin().cluster().cancelTasks(cancelTasksRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Updates status of the persistent task + */ + public void updateStatus(long taskId, Task.Status status, PersistentTaskOperationListener listener) { + UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status); + try { + client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, ActionListener.wrap( + o -> listener.onResponse(taskId), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Removes a persistent task + */ + public void removeTask(long taskId, PersistentTaskOperationListener listener) { + RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId); + try { + client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Starts a persistent task + */ + public void startTask(long taskId, PersistentTaskOperationListener listener) { + StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(taskId); + try { + client.execute(StartPersistentTaskAction.INSTANCE, startRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public interface PersistentTaskOperationListener { + void onResponse(long taskId); + void onFailure(Exception e); + } + +} diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java index 08dbd8a5e7afb..b4d636d258a96 100644 --- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -161,16 +161,16 @@ public final RequestBuilder setTaskId(long taskId) { public static class TransportAction extends TransportMasterNodeAction { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -191,7 +191,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTaskClusterService.removePersistentTask(request.taskId, new ActionListener() { + persistentTasksClusterService.removePersistentTask(request.taskId, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 8c302ab170e6e..6621088c41484 100644 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -44,7 +44,7 @@ import java.util.Objects; /** - * This action can be used to start persistent action previously created using {@link CreatePersistentTaskAction} + * This action can be used to start a persistent task previously created using {@link CreatePersistentTaskAction} */ public class StartPersistentTaskAction extends Action { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -194,7 +194,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTaskClusterService.startPersistentTask(request.taskId, new ActionListener() { + persistentTasksClusterService.startPersistentTask(request.taskId, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java index 332dbceb28773..d15b5ac8b54cf 100644 --- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -177,16 +177,16 @@ public final RequestBuilder setStatus(Task.Status status) { public static class TransportAction extends TransportMasterNodeAction { - private final PersistentTaskClusterService persistentTaskClusterService; + private final PersistentTasksClusterService persistentTasksClusterService; @Inject public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - PersistentTaskClusterService persistentTaskClusterService, + PersistentTasksClusterService persistentTasksClusterService, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); - this.persistentTaskClusterService = persistentTaskClusterService; + this.persistentTasksClusterService = persistentTasksClusterService; } @Override @@ -207,7 +207,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTaskClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener() { + persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/server/src/main/java/org/elasticsearch/persistent/package-info.java b/server/src/main/java/org/elasticsearch/persistent/package-info.java index ef8210f6db9cb..f948e3ace448e 100644 --- a/server/src/main/java/org/elasticsearch/persistent/package-info.java +++ b/server/src/main/java/org/elasticsearch/persistent/package-info.java @@ -18,31 +18,30 @@ */ /** - * The Persistent Actions are actions responsible for executing restartable actions that can survive disappearance of a + * The Persistent Tasks Executors are responsible for executing restartable tasks that can survive disappearance of a * coordinating and executor nodes. *

- * In order to be resilient to node restarts, the persistent actions are using the cluster state instead of a transport service to send + * In order to be resilient to node restarts, the persistent tasks are using the cluster state instead of a transport service to send * requests and responses. The execution is done in six phases: *

- * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent action. This action is handled - * by the {@link org.elasticsearch.persistent.PersistentActionService}, which is using - * {@link org.elasticsearch.persistent.PersistentTaskClusterService} to update cluster state with the record about running persistent + * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent task. This task is handled + * by the {@link org.elasticsearch.persistent.PersistentTasksService}, which is using + * {@link org.elasticsearch.persistent.PersistentTasksClusterService} to update cluster state with the record about running persistent * task. *

- * 2. The master node updates the {@link org.elasticsearch.persistent.PersistentTasks} in the cluster state to indicate that - * there is a new persistent action - * running in the system. + * 2. The master node updates the {@link org.elasticsearch.persistent.PersistentTasksCustomMetaData} in the cluster state to indicate + * that there is a new persistent task is running in the system. *

- * 3. The {@link org.elasticsearch.persistent.PersistentActionCoordinator} running on every node in the cluster monitors changes in - * the cluster state and starts execution of all new actions assigned to the node it is running on. + * 3. The {@link org.elasticsearch.persistent.PersistentTasksNodeService} running on every node in the cluster monitors changes in + * the cluster state and starts execution of all new tasks assigned to the node it is running on. *

- * 4. If the action fails to start on the node, the {@link org.elasticsearch.persistent.PersistentActionCoordinator} uses the - * {@link org.elasticsearch.persistent.PersistentTasks} to notify the - * {@link org.elasticsearch.persistent.PersistentActionService}, which reassigns the action to another node in the cluster. + * 4. If the task fails to start on the node, the {@link org.elasticsearch.persistent.PersistentTasksNodeService} uses the + * {@link org.elasticsearch.persistent.PersistentTasksCustomMetaData} to notify the + * {@link org.elasticsearch.persistent.PersistentTasksService}, which reassigns the action to another node in the cluster. *

- * 5. If action finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the - * cluster state unless . + * 5. If a task finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the + * cluster state unless removeOnCompletion flag for this task is set to false. *

- * 6. The {@link org.elasticsearch.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent action. + * 6. The {@link org.elasticsearch.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent task. */ package org.elasticsearch.persistent; diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java deleted file mode 100644 index efc48440cc971..0000000000000 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionRegistryTests.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.persistent; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.ThreadPool; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class PersistentActionRegistryTests extends ESTestCase { - - public void testActionLookup() { - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - TransportPersistentAction action1 = mock(TransportPersistentAction.class); - when(action1.getExecutor()).thenReturn(ThreadPool.Names.MANAGEMENT); - TransportPersistentAction action2 = mock(TransportPersistentAction.class); - when(action2.getExecutor()).thenReturn(ThreadPool.Names.GENERIC); - registry.registerPersistentAction("test1", action1); - registry.registerPersistentAction("test2", action2); - - assertEquals(registry.getPersistentActionHolderSafe("test1").getAction(), "test1"); - assertEquals(registry.getPersistentActionHolderSafe("test1").getExecutor(), ThreadPool.Names.MANAGEMENT); - assertEquals(registry.getPersistentActionHolderSafe("test1").getPersistentAction(), action1); - assertEquals(registry.getPersistentActionSafe("test1"), action1); - - assertEquals(registry.getPersistentActionHolderSafe("test2").getAction(), "test2"); - assertEquals(registry.getPersistentActionHolderSafe("test2").getExecutor(), ThreadPool.Names.GENERIC); - assertEquals(registry.getPersistentActionHolderSafe("test2").getPersistentAction(), action2); - assertEquals(registry.getPersistentActionSafe("test2"), action2); - - try { - registry.getPersistentActionHolderSafe("test3"); - fail("Should have failed"); - } catch (IllegalStateException ex) { - assertEquals(ex.getMessage(), "Unknown persistent action [test3]"); - } - - try { - registry.getPersistentActionSafe("test3"); - fail("Should have failed"); - } catch (IllegalStateException ex) { - assertEquals(ex.getMessage(), "Unknown persistent action [test3]"); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java similarity index 84% rename from server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index 466de5791089b..0337d6bc94b9e 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTaskClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -28,9 +28,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.persistent.PersistentTasks.Assignment; -import org.elasticsearch.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; import java.util.ArrayList; import java.util.Arrays; @@ -39,13 +39,13 @@ import java.util.List; import static java.util.Collections.emptyMap; -import static org.elasticsearch.persistent.TransportPersistentAction.NO_NODE_FOUND; +import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class PersistentTaskClusterServiceTests extends ESTestCase { +public class PersistentTasksClusterServiceTests extends ESTestCase { public void testReassignmentRequired() { int numberOfIterations = randomIntBetween(1, 30); @@ -60,10 +60,10 @@ public void testReassignmentRequired() { clusterState = insignificantChange(clusterState); } ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState); - assertThat(dumpEvent(event), PersistentTaskClusterService.reassignmentRequired(event, - new PersistentTaskClusterService.ExecutorNodeDecider() { + assertThat(dumpEvent(event), PersistentTasksClusterService.reassignmentRequired(event, + new PersistentTasksClusterService.ExecutorNodeDecider() { @Override - public Assignment getAssignment( + public Assignment getAssignment( String action, ClusterState currentState, Request request) { if ("never_assign".equals(((TestRequest) request).getTestParam())) { return NO_NODE_FOUND; @@ -76,14 +76,14 @@ public Assignment getAssignment( public void testReassignTasksWithNoTasks() { ClusterState clusterState = initialState(); - assertThat(reassign(clusterState).metaData().custom(PersistentTasks.TYPE), nullValue()); + assertThat(reassign(clusterState).metaData().custom(PersistentTasksCustomMetaData.TYPE), nullValue()); } public void testReassignConsidersClusterStateUpdates() { ClusterState clusterState = initialState(); ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks.Builder tasks = PersistentTasks.builder( - clusterState.metaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder( + clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(2, 40); @@ -91,11 +91,11 @@ public void testReassignConsidersClusterStateUpdates() { addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false); } - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); clusterState = builder.metaData(metaData).nodes(nodes).build(); ClusterState newClusterState = reassign(clusterState); - PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress, notNullValue()); } @@ -103,8 +103,8 @@ public void testReassignConsidersClusterStateUpdates() { public void testReassignTasks() { ClusterState clusterState = initialState(); ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks.Builder tasks = PersistentTasks.builder( - clusterState.metaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder( + clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(0, 40); @@ -128,11 +128,11 @@ public void testReassignTasks() { } } - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, tasks.build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); clusterState = builder.metaData(metaData).nodes(nodes).build(); ClusterState newClusterState = reassign(clusterState); - PersistentTasks tasksInProgress = newClusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress, notNullValue()); assertThat("number of tasks shouldn't change as a result or reassignment", @@ -143,15 +143,15 @@ public void testReassignTasks() { for (PersistentTask task : tasksInProgress.tasks()) { if (task.isStopped()) { assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue()); - assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getAction())); + assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getTaskName())); } else { // explanation should correspond to the action name - switch (task.getAction()) { + switch (task.getTaskName()) { case "should_assign": assertThat(task.getExecutorNode(), notNullValue()); assertThat(task.isAssigned(), equalTo(true)); if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { - logger.info(clusterState.metaData().custom(PersistentTasks.TYPE).toString()); + logger.info(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE).toString()); } assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); @@ -172,7 +172,7 @@ public void testReassignTasks() { } break; default: - fail("Unknown action " + task.getAction()); + fail("Unknown action " + task.getTaskName()); } } } @@ -186,10 +186,10 @@ private void addTestNodes(DiscoveryNodes.Builder nodes, int nonLocalNodesCount) } private ClusterState reassign(ClusterState clusterState) { - return PersistentTaskClusterService.reassignTasks(clusterState, logger, - new PersistentTaskClusterService.ExecutorNodeDecider() { + return PersistentTasksClusterService.reassignTasks(clusterState, logger, + new PersistentTasksClusterService.ExecutorNodeDecider() { @Override - public Assignment getAssignment( + public Assignment getAssignment( String action, ClusterState currentState, Request request) { TestRequest testRequest = (TestRequest) request; switch (testRequest.getTestParam()) { @@ -213,7 +213,7 @@ public Assignment getAssignment( private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { DiscoveryNodes nodes = clusterState.nodes(); - PersistentTasks tasksInProgress = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasksInProgress.findTasks("assign_one", task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) { return randomNodeAssignment(clusterState.nodes()); @@ -242,12 +242,12 @@ private String dumpEvent(ClusterChangedEvent event) { return "nodes_changed: " + event.nodesChanged() + " nodes_removed:" + event.nodesRemoved() + " routing_table_changed:" + event.routingTableChanged() + - " tasks: " + event.state().metaData().custom(PersistentTasks.TYPE); + " tasks: " + event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE); } private ClusterState significantChange(ClusterState clusterState) { ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks != null) { if (randomBoolean()) { for (PersistentTask task : tasks.tasks()) { @@ -265,11 +265,12 @@ private ClusterState significantChange(ClusterState clusterState) { // we don't have any unassigned tasks - add some if (randomBoolean()) { logger.info("added random task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), null, false); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), null, + false); tasksOrNodesChanged = true; } else { logger.info("added unassignable task with custom assignment message"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasks.builder(tasks), + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), new Assignment(null, "change me"), "never_assign", false); tasksOrNodesChanged = true; } @@ -292,10 +293,10 @@ private ClusterState significantChange(ClusterState clusterState) { return builder.build(); } - private PersistentTasks removeTasksWithChangingAssignment(PersistentTasks tasks) { + private PersistentTasksCustomMetaData removeTasksWithChangingAssignment(PersistentTasksCustomMetaData tasks) { if (tasks != null) { boolean changed = false; - PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); for (PersistentTask task : tasks.tasks()) { // Remove all unassigned tasks that cause changing assignments they might trigger a significant change if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) && @@ -314,9 +315,9 @@ private PersistentTasks removeTasksWithChangingAssignment(PersistentTasks tasks) private ClusterState insignificantChange(ClusterState clusterState) { ClusterState.Builder builder = ClusterState.builder(clusterState); - PersistentTasks tasks = clusterState.getMetaData().custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); tasks = removeTasksWithChangingAssignment(tasks); - PersistentTasks.Builder tasksBuilder = PersistentTasks.builder(tasks); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); if (randomBoolean()) { if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { @@ -338,7 +339,7 @@ private ClusterState insignificantChange(ClusterState clusterState) { } else { logger.info("changed routing table"); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); - metaData.putCustom(PersistentTasks.TYPE, tasksBuilder.build()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); changeRoutingTable(metaData, routingTable); builder.metaData(metaData).routingTable(routingTable.build()); @@ -360,12 +361,12 @@ private ClusterState insignificantChange(ClusterState clusterState) { // clear the task if (randomBoolean()) { logger.info("removed all tasks"); - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasks.TYPE, - PersistentTasks.builder().build()); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData.builder().build()); return builder.metaData(metaData).build(); } else { logger.info("set task custom to null"); - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasks.TYPE); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasksCustomMetaData.TYPE); return builder.metaData(metaData).build(); } } @@ -384,11 +385,11 @@ private ClusterState insignificantChange(ClusterState clusterState) { .numberOfReplicas(1) .build(); MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false) - .putCustom(PersistentTasks.TYPE, tasksBuilder.build()); + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); return builder.metaData(metaData).build(); } - private boolean hasAssignableTasks(PersistentTasks tasks, DiscoveryNodes discoveryNodes) { + private boolean hasAssignableTasks(PersistentTasksCustomMetaData tasks, DiscoveryNodes discoveryNodes) { if (tasks == null || tasks.tasks().isEmpty()) { return false; } @@ -403,26 +404,26 @@ private boolean hasAssignableTasks(PersistentTasks tasks, DiscoveryNodes discove }); } - private boolean hasTasksAssignedTo(PersistentTasks tasks, String nodeId) { + private boolean hasTasksAssignedTo(PersistentTasksCustomMetaData tasks, String nodeId) { return tasks != null && tasks.tasks().stream().anyMatch( task -> nodeId.equals(task.getExecutorNode())) == false; } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, - MetaData.Builder metaData, PersistentTasks.Builder tasks, + MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, String node, boolean stopped) { return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)), randomAsciiOfLength(10), stopped); } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, - MetaData.Builder metaData, PersistentTasks.Builder tasks, + MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, Assignment assignment, String param, boolean stopped) { - return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasks.TYPE, + return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build())); } - private void addTask(PersistentTasks.Builder tasks, String action, String param, String node, boolean stopped) { + private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node, boolean stopped) { tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action)); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java similarity index 82% rename from server/src/test/java/org/elasticsearch/persistent/PersistentTasksTests.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index caeb2583c3afa..30986386aeb29 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -34,12 +34,12 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractDiffableSerializationTestCase; -import org.elasticsearch.persistent.PersistentTasks.Assignment; -import org.elasticsearch.persistent.PersistentTasks.Builder; -import org.elasticsearch.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Builder; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; import java.io.IOException; import java.util.ArrayList; @@ -48,17 +48,17 @@ import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; -import static org.elasticsearch.persistent.TransportPersistentAction.NO_NODE_FOUND; +import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND; -public class PersistentTasksTests extends AbstractDiffableSerializationTestCase { +public class PersistentTasksCustomMetaDataTests extends AbstractDiffableSerializationTestCase { @Override - protected PersistentTasks createTestInstance() { + protected PersistentTasksCustomMetaData createTestInstance() { int numberOfTasks = randomInt(10); - PersistentTasks.Builder tasks = PersistentTasks.builder(); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); for (int i = 0; i < numberOfTasks; i++) { boolean stopped = randomBoolean(); - tasks.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), + tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment()); if (randomBoolean()) { // From time to time update status @@ -70,22 +70,22 @@ protected PersistentTasks createTestInstance() { @Override protected Writeable.Reader instanceReader() { - return PersistentTasks::new; + return PersistentTasksCustomMetaData::new; } @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Arrays.asList( - new Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), - new Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), - new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), + new Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new), + new Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom), + new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new), new Entry(Task.Status.class, Status.NAME, Status::new) )); } @Override protected Custom makeTestChanges(Custom testInstance) { - PersistentTasks tasksInProgress = (PersistentTasks) testInstance; + PersistentTasksCustomMetaData tasksInProgress = (PersistentTasksCustomMetaData) testInstance; Builder builder = new Builder(); switch (randomInt(3)) { case 0: @@ -118,12 +118,12 @@ protected Custom makeTestChanges(Custom testInstance) { @Override protected Writeable.Reader> diffReader() { - return PersistentTasks::readDiffFrom; + return PersistentTasksCustomMetaData::readDiffFrom; } @Override - protected PersistentTasks doParseInstance(XContentParser parser) throws IOException { - return PersistentTasks.fromXContent(parser); + protected PersistentTasksCustomMetaData doParseInstance(XContentParser parser) throws IOException { + return PersistentTasksCustomMetaData.fromXContent(parser); } @Override @@ -150,19 +150,19 @@ protected XContentBuilder toXContent(Custom instance, XContentType contentType, private Builder addRandomTask(Builder builder) { boolean stopped = randomBoolean(); - builder.addTask(TestPersistentAction.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), + builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment()); return builder; } - private long pickRandomTask(PersistentTasks testInstance) { + private long pickRandomTask(PersistentTasksCustomMetaData testInstance) { return randomFrom(new ArrayList<>(testInstance.tasks())).getId(); } @Override protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(Arrays.asList( - new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), + new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME), TestRequest::fromXContent), new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) )); @@ -170,9 +170,9 @@ protected NamedXContentRegistry xContentRegistry() { @SuppressWarnings("unchecked") public void testSerializationContext() throws Exception { - PersistentTasks testInstance = createTestInstance(); + PersistentTasksCustomMetaData testInstance = createTestInstance(); for (int i = 0; i < randomInt(10); i++) { - testInstance = (PersistentTasks) makeTestChanges(testInstance); + testInstance = (PersistentTasksCustomMetaData) makeTestChanges(testInstance); } ToXContent.MapParams params = new ToXContent.MapParams( @@ -183,7 +183,7 @@ public void testSerializationContext() throws Exception { XContentBuilder shuffled = shuffleXContent(builder); XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes()); - PersistentTasks newInstance = doParseInstance(parser); + PersistentTasksCustomMetaData newInstance = doParseInstance(parser); assertNotSame(newInstance, testInstance); assertEquals(testInstance.tasks().size(), newInstance.tasks().size()); @@ -192,7 +192,7 @@ public void testSerializationContext() throws Exception { assertNotNull(newTask); // Things that should be serialized - assertEquals(testTask.getAction(), newTask.getAction()); + assertEquals(testTask.getTaskName(), newTask.getTaskName()); assertEquals(testTask.getId(), newTask.getId()); assertEquals(testTask.getStatus(), newTask.getStatus()); assertEquals(testTask.getRequest(), newTask.getRequest()); @@ -205,7 +205,7 @@ public void testSerializationContext() throws Exception { } public void testBuilder() { - PersistentTasks persistentTasks = null; + PersistentTasksCustomMetaData persistentTasks = null; long lastKnownTask = -1; for (int i = 0; i < randomIntBetween(10, 100); i++) { final Builder builder; diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java similarity index 62% rename from server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index 4da92332dd14e..1c23504dc5662 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -17,14 +17,16 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.persistent.PersistentTasks.PersistentTask; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.persistent.PersistentTasksExecutorIT.PersistentTaskOperationFuture; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -32,10 +34,10 @@ import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1) -public class PersistentActionFullRestartIT extends ESIntegTestCase { +public class PersistentTasksExecutorFullRestartIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singletonList(TestPersistentActionPlugin.class); + return Collections.singletonList(TestPersistentTasksPlugin.class); } @Override @@ -49,35 +51,38 @@ protected boolean ignoreExternalCluster() { @TestLogging("org.elasticsearch.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG") public void testFullClusterRestart() throws Exception { + PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class); int numberOfTasks = randomIntBetween(1, 10); long[] taskIds = new long[numberOfTasks]; + List futures = new ArrayList<>(numberOfTasks); + boolean[] stopped = new boolean[numberOfTasks]; int runningTasks = 0; for (int i = 0; i < numberOfTasks; i++) { - if (randomBoolean()) { + stopped[i] = randomBoolean(); + if (stopped[i] == false) { runningTasks++; - taskIds[i] = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); - stopped[i] = false; - } else { - taskIds[i] = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client()) - .setAction(TestPersistentAction.NAME) - .setRequest(new TestRequest("Blah")) - .setStopped(true) - .get().getTaskId(); - stopped[i] = true; } + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + futures.add(future); + service.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped[i], true, future); } + + for (int i = 0; i < numberOfTasks; i++) { + taskIds[i] = futures.get(i).get(); + } + final int numberOfRunningTasks = runningTasks; - PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); if (numberOfRunningTasks > 0) { // Make sure that at least one of the tasks is running assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() - .getTasks().size(), greaterThan(0)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), greaterThan(0)); }); } @@ -85,7 +90,7 @@ public void testFullClusterRestart() throws Exception { internalCluster().fullRestart(); ensureYellow(); - tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); // Check that cluster state is correct for (int i = 0; i < numberOfTasks; i++) { @@ -97,12 +102,13 @@ public void testFullClusterRestart() throws Exception { logger.info("Waiting for {} original tasks to start", numberOfRunningTasks); assertBusy(() -> { // Wait for the running task to start automatically - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(numberOfRunningTasks)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(numberOfRunningTasks)); }); // Start all other tasks - tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasks.TYPE); + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + service = internalCluster().getInstance(PersistentTasksService.class); for (int i = 0; i < numberOfTasks; i++) { PersistentTask task = tasksInProgress.getTask(taskIds[i]); assertNotNull(task); @@ -110,26 +116,28 @@ public void testFullClusterRestart() throws Exception { assertThat(task.isStopped(), equalTo(stopped[i])); assertThat(task.getExecutorNode(), stopped[i] ? nullValue() : notNullValue()); if (stopped[i]) { - assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(task.getId()).get()); + PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture(); + service.startTask(task.getId(), startFuture); + assertEquals(startFuture.get(), (Long) task.getId()); } } logger.info("Waiting for {} tasks to start", numberOfTasks); assertBusy(() -> { // Wait for all tasks to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(numberOfTasks)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(numberOfTasks)); }); logger.info("Complete all tasks"); // Complete the running task and make sure it finishes properly - assertThat(new TestPersistentActionPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(), + assertThat(new TestPersistentTasksPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(), equalTo(numberOfTasks)); assertBusy(() -> { // Make sure the task is removed from the cluster state - assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE)).tasks(), empty()); + assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty()); }); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java similarity index 62% rename from server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index be8b241b7004f..0ca059cc74aa8 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -20,12 +20,15 @@ package org.elasticsearch.persistent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.BaseFuture; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestTasksRequestBuilder; +import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder; import org.junit.After; import java.util.Collection; @@ -40,11 +43,11 @@ import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2) -public class PersistentActionIT extends ESIntegTestCase { +public class PersistentTasksExecutorIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singletonList(TestPersistentActionPlugin.class); + return Collections.singletonList(TestPersistentTasksPlugin.class); } @Override @@ -61,14 +64,30 @@ public void cleanup() throws Exception { assertNoRunningTasks(); } + public static class PersistentTaskOperationFuture extends BaseFuture implements PersistentTaskOperationListener { + + @Override + public void onResponse(long taskId) { + set(taskId); + } + + @Override + public void onFailure(Exception e) { + setException(e); + } + } + public void testPersistentActionRestart() throws Exception { - long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get(); assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); // Verifying parent @@ -82,7 +101,7 @@ public void testPersistentActionRestart() throws Exception { assertBusy(() -> { // Wait for the task to restart - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() .getTasks(); logger.info("Found {} tasks", tasks.size()); assertThat(tasks.size(), equalTo(1)); @@ -92,24 +111,29 @@ public void testPersistentActionRestart() throws Exception { logger.info("Removing persistent task with id {}", firstRunningTask.getId()); // Remove the persistent task - assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); + persistentTasksService.removeTask(taskId, removeFuture); + assertEquals(removeFuture.get(), (Long) taskId); logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); assertBusy(() -> { // Wait for the task to disappear completely - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), empty()); }); } public void testPersistentActionCompletion() throws Exception { - long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get().getTaskId(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get(); assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); // Verifying parent @@ -120,14 +144,14 @@ public void testPersistentActionCompletion() throws Exception { public void testPersistentActionCompletionWithoutRemoval() throws Exception { boolean stopped = randomBoolean(); - long taskId = CreatePersistentTaskAction.INSTANCE.newRequestBuilder(client()) - .setAction(TestPersistentAction.NAME) - .setRequest(new TestPersistentActionPlugin.TestRequest("Blah")) - .setRemoveOnCompletion(false) - .setStopped(stopped).get().getTaskId(); - - PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped, false, + future); + long taskId = future.get(); + + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped)); assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue()); @@ -138,24 +162,26 @@ public void testPersistentActionCompletionWithoutRemoval() throws Exception { for (int i = 0; i < numberOfIters; i++) { logger.info("iteration {}", i); if (stopped) { - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), - empty()); - assertAcked(StartPersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks(), empty()); + PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture(); + persistentTasksService.startTask(taskId, startFuture); + assertEquals(startFuture.get(), (Long) taskId); } assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks() - .size(), equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); stopOrCancelTask(firstRunningTask.getTaskId()); assertBusy(() -> { // Wait for the task to finish - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() - .getTasks(); + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .get().getTasks(); logger.info("Found {} tasks", tasks.size()); assertThat(tasks.size(), equalTo(0)); }); @@ -164,30 +190,36 @@ public void testPersistentActionCompletionWithoutRemoval() throws Exception { assertBusy(() -> { // Wait for the task to be marked as stopped - PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.getTask(taskId).isStopped(), equalTo(true)); assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); }); logger.info("Removing action record from cluster state"); - assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); + PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); + persistentTasksService.removeTask(taskId, removeFuture); + assertEquals(removeFuture.get(), (Long) taskId); } public void testPersistentActionWithNoAvailableNode() throws Exception { - long taskId = TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah") - .executorNodeAttr("test").get().getTaskId(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + TestRequest testRequest = new TestRequest("Blah"); + testRequest.setExecutorNodeAttr("test"); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, testRequest, future); + long taskId = future.get(); Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); String newNode = internalCluster().startNode(nodeSettings); String newNodeId = internalCluster().clusterService(newNode).localNode().getId(); assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() + .size(), equalTo(1)); }); - TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); // Verifying the the task runs on the new node @@ -197,27 +229,32 @@ public void testPersistentActionWithNoAvailableNode() throws Exception { assertBusy(() -> { // Wait for the task to disappear completely - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks(), + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), empty()); }); // Remove the persistent task - assertAcked(RemovePersistentTaskAction.INSTANCE.newRequestBuilder(client()).setTaskId(taskId).get()); - + PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); + persistentTasksService.removeTask(taskId, removeFuture); + assertEquals(removeFuture.get(), (Long) taskId); } public void testPersistentActionStatusUpdate() throws Exception { - TestPersistentAction.INSTANCE.newRequestBuilder(client()).testParam("Blah").get(); + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + future.get(); + assertBusy(() -> { // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get().getTasks().size(), - equalTo(1)); + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() + .size(), equalTo(1)); }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]") + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); - PersistentTasks tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(1)); assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue()); @@ -230,8 +267,8 @@ public void testPersistentActionStatusUpdate() throws Exception { int finalI = i; assertBusy(() -> { - PersistentTasks tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE); + PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasks.tasks().size(), equalTo(1)); assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue()); assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); @@ -266,14 +303,14 @@ private void stopOrCancelTask(TaskId taskId) { private void assertNoRunningTasks() throws Exception { assertBusy(() -> { // Wait for the task to finish - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentAction.NAME + "[c]").get() + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() .getTasks(); logger.info("Found {} tasks", tasks.size()); assertThat(tasks.size(), equalTo(0)); // Make sure the task is removed from the cluster state - assertThat(((PersistentTasks) internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasks.TYPE)).tasks(), empty()); + assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty()); }); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java similarity index 73% rename from server/src/test/java/org/elasticsearch/persistent/PersistentActionResponseTests.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java index 2a1f24dabed40..691077c80f2f4 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionResponseTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java @@ -20,15 +20,15 @@ import org.elasticsearch.test.AbstractStreamableTestCase; -public class PersistentActionResponseTests extends AbstractStreamableTestCase { +public class PersistentTasksExecutorResponseTests extends AbstractStreamableTestCase { @Override - protected PersistentActionResponse createTestInstance() { - return new PersistentActionResponse(randomLong()); + protected PersistentTaskResponse createTestInstance() { + return new PersistentTaskResponse(randomLong()); } @Override - protected PersistentActionResponse createBlankInstance() { - return new PersistentActionResponse(); + protected PersistentTaskResponse createBlankInstance() { + return new PersistentTaskResponse(); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorStatusTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java similarity index 82% rename from server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorStatusTests.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java index fbb649d17fce5..8bb34297b078a 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorStatusTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java @@ -16,12 +16,12 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.persistent.PersistentActionCoordinator.State; -import org.elasticsearch.persistent.PersistentActionCoordinator.Status; +import org.elasticsearch.persistent.PersistentTasksNodeService.State; +import org.elasticsearch.persistent.PersistentTasksNodeService.Status; import static org.hamcrest.Matchers.containsString; -public class PersistentActionCoordinatorStatusTests extends AbstractWireSerializingTestCase { +public class PersistentTasksNodeServiceStatusTests extends AbstractWireSerializingTestCase { @Override protected Status createTestInstance() { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java similarity index 59% rename from server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java rename to server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 10ebb4fa2703a..c5dba05a12f62 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentActionCoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; @@ -36,12 +35,13 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.persistent.CompletionPersistentTaskAction.Response; -import org.elasticsearch.persistent.PersistentTasks.Assignment; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -51,7 +51,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class PersistentActionCoordinatorTests extends ESTestCase { +public class PersistentTasksNodeServiceTests extends ESTestCase { private ClusterService createClusterService() { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); @@ -72,21 +72,21 @@ private DiscoveryNodes createTestNodes(int nonLocalNodesCount, Settings settings public void testStartTask() throws Exception { ClusterService clusterService = createClusterService(); - PersistentActionService persistentActionService = mock(PersistentActionService.class); - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - registry.registerPersistentAction("test", action); + when(action.getTaskName()).thenReturn("test"); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); int nonLocalNodesCount = randomInt(10); MockExecutor executor = new MockExecutor(); - PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, - registry, new TaskManager(Settings.EMPTY), executor); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, new TaskManager(Settings.EMPTY), mock(ThreadPool.class), executor); ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); - PersistentTasks.Builder tasks = PersistentTasks.builder(); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { @@ -105,7 +105,7 @@ public void testStartTask() throws Exception { } MetaData.Builder metaData = MetaData.builder(state.metaData()); - metaData.putCustom(PersistentTasks.TYPE, tasks.build()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); @@ -163,24 +163,24 @@ public void testStartTask() throws Exception { public void testTaskCancellation() { ClusterService clusterService = createClusterService(); AtomicLong capturedTaskId = new AtomicLong(); - AtomicReference> capturedListener = new AtomicReference<>(); - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, null, null, null) { + AtomicReference capturedListener = new AtomicReference<>(); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null) { @Override - public void sendCancellation(long taskId, ActionListener listener) { + public void sendCancellation(long taskId, PersistentTaskOperationListener listener) { capturedTaskId.set(taskId); capturedListener.set(listener); } }; - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - registry.registerPersistentAction("test", action); + when(action.getTaskName()).thenReturn("test"); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); int nonLocalNodesCount = randomInt(10); MockExecutor executor = new MockExecutor(); TaskManager taskManager = new TaskManager(Settings.EMPTY); - PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, - registry, taskManager, executor); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, taskManager, mock(ThreadPool.class), executor); ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); @@ -217,7 +217,7 @@ public void sendCancellation(long taskId, ActionListener li // That should trigger cancellation request assertThat(capturedTaskId.get(), equalTo(localId)); // Notify successful cancellation - capturedListener.get().onResponse(new CancelTasksResponse()); + capturedListener.get().onResponse(localId); // finish or fail task if (randomBoolean()) { @@ -231,106 +231,118 @@ public void sendCancellation(long taskId, ActionListener li } - public void testNotificationFailure() { - ClusterService clusterService = createClusterService(); - AtomicLong capturedTaskId = new AtomicLong(-1L); - AtomicReference capturedException = new AtomicReference<>(); - AtomicReference> capturedListener = new AtomicReference<>(); - PersistentActionService persistentActionService = - new PersistentActionService(Settings.EMPTY, mock(ThreadPool.class), clusterService, null) { - @Override - public void sendCompletionNotification(long taskId, Exception failure, ActionListener listener) { - capturedTaskId.set(taskId); - capturedException.set(failure); - capturedListener.set(listener); - } - }; - PersistentActionRegistry registry = new PersistentActionRegistry(Settings.EMPTY); - @SuppressWarnings("unchecked") TransportPersistentAction action = mock(TransportPersistentAction.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - registry.registerPersistentAction("test", action); - - int nonLocalNodesCount = randomInt(10); - MockExecutor executor = new MockExecutor(); - TaskManager taskManager = new TaskManager(Settings.EMPTY); - PersistentActionCoordinator coordinator = new PersistentActionCoordinator(Settings.EMPTY, persistentActionService, - registry, taskManager, executor); - - ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) + public void testNotificationFailure() throws Exception { + Settings settings = Settings.builder() + .put("node.name", PersistentTasksNodeServiceTests.class.getSimpleName()) .build(); + ThreadPool threadPool = new ThreadPool(settings); + try { + ClusterService clusterService = createClusterService(); + AtomicLong capturedTaskId = new AtomicLong(-1L); + AtomicReference capturedException = new AtomicReference<>(); + AtomicReference capturedListener = new AtomicReference<>(); + PersistentTasksService persistentTasksService = + new PersistentTasksService(Settings.EMPTY, clusterService, null) { + @Override + public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) { + capturedTaskId.set(taskId); + capturedException.set(failure); + capturedListener.set(listener); + } + }; + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getTaskName()).thenReturn("test"); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, + Collections.singletonList(action)); + + int nonLocalNodesCount = randomInt(10); + MockExecutor executor = new MockExecutor(); + TaskManager taskManager = new TaskManager(Settings.EMPTY); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, taskManager, threadPool, executor); + + ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) + .build(); + + ClusterState newClusterState = state; + // Allocate first task + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - ClusterState newClusterState = state; - // Allocate first task - state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "this_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - - // Fail the task - executor.get(0).listener.onFailure(new RuntimeException("test failure")); + // Fail the task + executor.get(0).listener.onFailure(new RuntimeException("test failure")); - // Check that notification was sent - assertThat(capturedException.get().getMessage(), equalTo("test failure")); - capturedException.set(null); + // Check that notification was sent + assertThat(capturedException.get().getMessage(), equalTo("test failure")); + capturedException.set(null); - // Simulate failure to notify - capturedListener.get().onFailure(new IOException("simulated notification failure")); + // Simulate failure to notify + capturedListener.get().onFailure(new IOException("simulated notification failure")); - // Allocate another task - state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "other_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + // Allocate another task + state = newClusterState; + newClusterState = addTask(state, "test", new TestRequest(), "other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - // Check that notification was sent again - assertThat(capturedException.get().getMessage(), equalTo("test failure")); + // Check that notification was sent again + assertBusy(() -> { + assertNotNull(capturedException.get()); + assertThat(capturedException.get().getMessage(), equalTo("test failure")); + }); - // Check the the task is still known by the task manager - assertThat(taskManager.getTasks().size(), equalTo(1)); - long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); + // Check the the task is still known by the task manager + assertThat(taskManager.getTasks().size(), equalTo(1)); + long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); - // This time acknowledge notification - capturedListener.get().onResponse(new Response()); + // This time acknowledge notification + capturedListener.get().onResponse(id); - // Reallocate failed task to another node - state = newClusterState; - newClusterState = reallocateTask(state, id, "other_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + // Reallocate failed task to another node + state = newClusterState; + newClusterState = reallocateTask(state, id, "other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - // Check the the task is now removed from task manager - assertThat(taskManager.getTasks().values(), empty()); + // Check the the task is now removed from task manager + assertThat(taskManager.getTasks().values(), empty()); + } finally { + assertTrue(ESTestCase.terminate(threadPool)); + } } - private ClusterState addTask(ClusterState state, String action, Request request, - String node) { - PersistentTasks.Builder builder = - PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, + private ClusterState addTask(ClusterState state, String action, Request request, + String node) { + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build(); } private ClusterState reallocateTask(ClusterState state, long taskId, String node) { - PersistentTasks.Builder builder = - PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); assertTrue(builder.hasTask(taskId)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build(); } private ClusterState removeTask(ClusterState state, long taskId) { - PersistentTasks.Builder builder = - PersistentTasks.builder(state.getMetaData().custom(PersistentTasks.TYPE)); + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); assertTrue(builder.hasTask(taskId)); - return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasks.TYPE, + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, builder.removeTask(taskId).build())).build(); } private class Execution { - private final PersistentActionRequest request; + private final PersistentTaskRequest request; private final NodePersistentTask task; - private final PersistentActionRegistry.PersistentActionHolder holder; + private final PersistentTasksExecutor holder; private final ActionListener listener; - Execution(PersistentActionRequest request, NodePersistentTask task, PersistentActionRegistry.PersistentActionHolder holder, + Execution(PersistentTaskRequest request, NodePersistentTask task, PersistentTasksExecutor holder, ActionListener listener) { this.request = request; this.task = task; @@ -339,7 +351,7 @@ private class Execution { } } - private class MockExecutor extends PersistentActionExecutor { + private class MockExecutor extends NodePersistentTasksExecutor { private List executions = new ArrayList<>(); MockExecutor() { @@ -347,10 +359,10 @@ private class MockExecutor extends PersistentActionExecutor { } @Override - public void executeAction(Request request, NodePersistentTask task, - PersistentActionRegistry.PersistentActionHolder holder, - ActionListener listener) { - executions.add(new Execution(request, task, holder, listener)); + public void executeTask(Request request, NodePersistentTask task, + PersistentTasksExecutor action, + ActionListener listener) { + executions.add(new Execution(request, task, action, listener)); } public Execution get(int i) { diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java index 17d8ca6954323..52b3c245ef490 100644 --- a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -21,8 +21,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.persistent.CreatePersistentTaskAction.Request; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestPersistentAction; -import org.elasticsearch.persistent.TestPersistentActionPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; import org.elasticsearch.test.AbstractStreamableTestCase; import java.util.Collections; @@ -52,7 +52,7 @@ protected Request createBlankInstance() { @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Collections.singletonList( - new Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new) + new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new) )); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java similarity index 82% rename from server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java rename to server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 2c97bea928de6..040dadaf1ac93 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentActionPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.FailedNodeException; @@ -63,7 +62,7 @@ import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.persistent.PersistentTasks.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import java.io.IOException; import java.util.ArrayList; @@ -86,12 +85,11 @@ /** * A plugin that adds a test persistent task. */ -public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { +public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { return Arrays.asList( - new ActionHandler<>(TestPersistentAction.INSTANCE, TransportTestPersistentAction.class), new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class), new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), @@ -105,24 +103,28 @@ public class TestPersistentActionPlugin extends Plugin implements ActionPlugin { public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry) { - - PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, threadPool, clusterService, client); - PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, client); + TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, persistentTasksService, + clusterService); + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, + Collections.singletonList(testPersistentAction)); return Arrays.asList( - persistentActionService, - persistentActionRegistry, - new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService) + persistentTasksService, + persistentTasksExecutorRegistry, + new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService) ); } @Override public List getNamedWriteables() { return Arrays.asList( - new NamedWriteableRegistry.Entry(PersistentActionRequest.class, TestPersistentAction.NAME, TestRequest::new), + new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new), new NamedWriteableRegistry.Entry(Task.Status.class, - PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), - new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasks.TYPE, PersistentTasks::new), - new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasks.TYPE, PersistentTasks::readDiffFrom), + PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::readDiffFrom), new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) ); } @@ -130,18 +132,18 @@ public List getNamedWriteables() { @Override public List getNamedXContent() { return Arrays.asList( - new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasks.TYPE), - PersistentTasks::fromXContent), - new NamedXContentRegistry.Entry(PersistentActionRequest.class, new ParseField(TestPersistentAction.NAME), + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE), + PersistentTasksCustomMetaData::fromXContent), + new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME), TestRequest::fromXContent), new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) ); } - public static class TestRequest extends PersistentActionRequest { + public static class TestRequest extends PersistentTaskRequest { public static final ConstructingObjectParser REQUEST_PARSER = - new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new TestRequest((String) args[0])); + new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new TestRequest((String) args[0])); static { REQUEST_PARSER.declareString(constructorArg(), new ParseField("param")); @@ -172,7 +174,7 @@ public ActionRequestValidationException validate() { @Override public String getWriteableName() { - return TestPersistentAction.NAME; + return TestPersistentTasksExecutor.NAME; } public void setExecutorNodeAttr(String executorNodeAttr) { @@ -240,53 +242,13 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId) } } - public static class TestPersistentTaskRequestBuilder extends - ActionRequestBuilder { - - protected TestPersistentTaskRequestBuilder(ElasticsearchClient client, Action action, TestRequest request) { - super(client, action, request); - } - - public TestPersistentTaskRequestBuilder testParam(String testParam) { - request.setTestParam(testParam); - return this; - } - - public TestPersistentTaskRequestBuilder executorNodeAttr(String targetNode) { - request.setExecutorNodeAttr(targetNode); - return this; - } - - } - - public static class TestPersistentAction extends Action { - - public static final TestPersistentAction INSTANCE = new TestPersistentAction(); - public static final String NAME = "cluster:admin/persistent/test"; - - private TestPersistentAction() { - super(NAME); - } - - @Override - public PersistentActionResponse newResponse() { - return new PersistentActionResponse(); - } - - @Override - public TestPersistentTaskRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new TestPersistentTaskRequestBuilder(client, this, new TestRequest()); - } - } - public static class Status implements Task.Status { public static final String NAME = "test"; private final String phase; public static final ConstructingObjectParser STATUS_PARSER = - new ConstructingObjectParser<>(TestPersistentAction.NAME, args -> new Status((String) args[0])); + new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new Status((String) args[0])); static { STATUS_PARSER.declareString(constructorArg(), new ParseField("phase")); @@ -350,19 +312,15 @@ public int hashCode() { } - public static class TransportTestPersistentAction extends TransportPersistentAction { + public static class TestPersistentTasksExecutor extends PersistentTasksExecutor { - private final TransportService transportService; + public static final String NAME = "cluster:admin/persistent/test"; + private final ClusterService clusterService; - @Inject - public TransportTestPersistentAction(Settings settings, ThreadPool threadPool, TransportService transportService, - PersistentActionService persistentActionService, - PersistentActionRegistry persistentActionRegistry, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, TestPersistentAction.NAME, false, threadPool, transportService, persistentActionService, - persistentActionRegistry, actionFilters, indexNameExpressionResolver, TestRequest::new, - ThreadPool.Names.GENERIC); - this.transportService = transportService; + public TestPersistentTasksExecutor(Settings settings, PersistentTasksService persistentTasksService, + ClusterService clusterService) { + super(settings, NAME, persistentTasksService, ThreadPool.Names.GENERIC); + this.clusterService = clusterService; } @Override @@ -391,9 +349,9 @@ protected void nodeOperation(NodePersistentTask task, TestRequest request, Actio // wait for something to happen assertTrue(awaitBusy(() -> testTask.isCancelled() || testTask.getOperation() != null || - transportService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes + clusterService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes 30, TimeUnit.SECONDS)); // This can take a while during large cluster restart - if (transportService.lifecycleState() != Lifecycle.State.STARTED) { + if (clusterService.lifecycleState() != Lifecycle.State.STARTED) { return; } if ("finish".equals(testTask.getOperation())) { diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java index 751286eafa09d..3cf9e8bbc6245 100644 --- a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractStreamableTestCase; -import org.elasticsearch.persistent.TestPersistentActionPlugin.Status; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction.Request; import java.util.Collections; From 19f39fd39233aef6210250b358bda350aaf245a1 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 23 Mar 2017 12:56:48 -0400 Subject: [PATCH 13/50] Persistent Tasks: remove task restart on failure (#815) If a persistent task throws an exception, the persistent tasks framework will no longer try to restart the task. This is a temporary measure to prevent threshing the cluster with endless restart attempt. We will revisit this in the future version to make the restart process more robust. Please note, however, that if node executing the task goes down, the task will still be restarted on another node. --- .../CompletionPersistentTaskAction.java | 2 +- .../PersistentTasksClusterService.java | 15 +++++---------- .../PersistentTasksCustomMetaData.java | 19 ++----------------- .../PersistentTasksClusterServiceTests.java | 4 ++++ .../PersistentTasksCustomMetaDataTests.java | 6 +----- .../PersistentTasksExecutorFullRestartIT.java | 4 ++++ .../persistent/PersistentTasksExecutorIT.java | 18 +----------------- ...PersistentTasksNodeServiceStatusTests.java | 6 +++++- 8 files changed, 23 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java index 6a76774895003..95b81cfdea45d 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -191,7 +191,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener() { + persistentTasksClusterService.completePersistentTask(request.taskId, request.exception, new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(newResponse()); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index a66acc2bbbf81..8203ff3396c88 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -99,25 +99,20 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @param failure the reason for restarting the task or null if the task completed successfully * @param listener the listener that will be called when task is removed */ - public void completeOrRestartPersistentTask(long id, Exception failure, ActionListener listener) { + public void completePersistentTask(long id, Exception failure, ActionListener listener) { final String source; if (failure != null) { - logger.warn("persistent task " + id + " failed, restarting", failure); - source = "restart persistent task"; + logger.warn("persistent task " + id + " failed", failure); + source = "finish persistent task (failed)"; } else { - source = "finish persistent task"; + source = "finish persistent task (success)"; } clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); if (tasksInProgress.hasTask(id)) { - if (failure != null) { - // If the task failed - we need to restart it on another node, otherwise we just remove it - tasksInProgress.reassignTask(id, (action, request) -> getAssignement(action, currentState, request)); - } else { - tasksInProgress.finishTask(id); - } + tasksInProgress.finishTask(id); return update(currentState, tasksInProgress); } else { // we don't send the error message back to the caller becase that would cause an infinite loop of notifications diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index c78cb07e8c764..3805a510c394e 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -172,7 +172,7 @@ public long getNumberOfTasksOnNode(String nodeId, String taskName) { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_3_0_UNRELEASED; + return Version.V_5_4_0_UNRELEASED; } @Override @@ -601,8 +601,7 @@ public Builder reassignTask(long taskId, Assignment assignment) { /** * Assigns the task to another node if the task exist and not currently assigned *

- * The operation is only performed if the task is not currently assigned to any nodes. To force assignment use - * {@link #reassignTask(long, BiFunction)} instead + * The operation is only performed if the task is not currently assigned to any nodes. */ @SuppressWarnings("unchecked") public Builder assignTask(long taskId, @@ -618,20 +617,6 @@ public Builder assignTask(long taskId, return this; } - /** - * Reassigns the task to another node if the task exist - */ - @SuppressWarnings("unchecked") - public Builder reassignTask(long taskId, - BiFunction executorNodeFunc) { - PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); - if (taskInProgress != null) { - changed = true; - Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request); - tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); - } - return this; - } /** * Updates the task status if the task exist diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index 0337d6bc94b9e..fd538f8d6ec01 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -11,6 +11,10 @@ * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.elasticsearch.persistent; diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index 30986386aeb29..dfeb7152e1cf5 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -225,11 +225,7 @@ public void testBuilder() { if (builder.hasTask(lastKnownTask)) { changed = true; } - if (randomBoolean()) { - builder.reassignTask(lastKnownTask, randomAssignment()); - } else { - builder.reassignTask(lastKnownTask, (s, request) -> randomAssignment()); - } + builder.reassignTask(lastKnownTask, randomAssignment()); break; case 2: if (builder.hasTask(lastKnownTask)) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index 1c23504dc5662..a13d683673fe5 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -11,6 +11,10 @@ * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.elasticsearch.persistent; diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 0ca059cc74aa8..87793860caa1f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -77,7 +77,7 @@ public void onFailure(Exception e) { } } - public void testPersistentActionRestart() throws Exception { + public void testPersistentActionFailure() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); @@ -99,22 +99,6 @@ public void testPersistentActionRestart() throws Exception { assertThat(new TestTasksRequestBuilder(client()).setOperation("fail").setTaskId(firstRunningTask.getTaskId()) .get().getTasks().size(), equalTo(1)); - assertBusy(() -> { - // Wait for the task to restart - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() - .getTasks(); - logger.info("Found {} tasks", tasks.size()); - assertThat(tasks.size(), equalTo(1)); - // Make sure that restarted task is different - assertThat(tasks.get(0).getTaskId(), not(equalTo(firstRunningTask.getTaskId()))); - }); - - logger.info("Removing persistent task with id {}", firstRunningTask.getId()); - // Remove the persistent task - PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); - persistentTasksService.removeTask(taskId, removeFuture); - assertEquals(removeFuture.get(), (Long) taskId); - logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); assertBusy(() -> { // Wait for the task to disappear completely diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java index 8bb34297b078a..9d831b5563560 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java @@ -11,6 +11,10 @@ * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.elasticsearch.persistent; @@ -36,4 +40,4 @@ protected Writeable.Reader instanceReader() { public void testToString() { assertThat(createTestInstance().toString(), containsString("state")); } -} \ No newline at end of file +} From 37fad048794707d37e7c97f478a06d3c42a41f3e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Mar 2017 14:21:01 -0400 Subject: [PATCH 14/50] Persistent Tasks: Merge NodePersistentTask and RunningPersistentTask (#842) Refactors NodePersistentTask and RunningPersistentTask into a single AllocatedPersistentTask. Makes it possible to update Persistent Task Status via AllocatedPersistentTask. --- .../persistent/AllocatedPersistentTask.java | 124 +++++++++++++ .../persistent/NodePersistentTask.java | 74 -------- .../NodePersistentTasksExecutor.java | 2 +- .../persistent/PersistentTaskRequest.java | 2 +- .../persistent/PersistentTasksExecutor.java | 4 +- .../PersistentTasksNodeService.java | 168 +++++------------- ...PersistentTasksNodeServiceStatusTests.java | 3 +- .../PersistentTasksNodeServiceTests.java | 6 +- .../persistent/TestPersistentTasksPlugin.java | 4 +- 9 files changed, 180 insertions(+), 207 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java delete mode 100644 server/src/main/java/org/elasticsearch/persistent/NodePersistentTask.java diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java new file mode 100644 index 0000000000000..6bad551f98494 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * Represents a executor node operation that corresponds to a persistent task + */ +public class AllocatedPersistentTask extends CancellableTask { + private long persistentTaskId; + + private final AtomicReference state; + @Nullable + private Exception failure; + + private PersistentTasksService persistentTasksService; + + + public AllocatedPersistentTask(long id, String type, String action, String description, TaskId parentTask) { + super(id, type, action, description, parentTask); + this.state = new AtomicReference<>(State.STARTED); + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + + // In case of persistent tasks we always need to return: `false` + // because in case of persistent task the parent task isn't a task in the task manager, but in cluster state. + // This instructs the task manager not to try to kill this persistent task when the task manager cannot find + // a fake parent node id "cluster" in the cluster state + @Override + public final boolean cancelOnParentLeaving() { + return false; + } + + @Override + public Status getStatus() { + return new PersistentTasksNodeService.Status(state.get()); + } + + /** + * Updates the persistent state for the corresponding persistent task. + * + * This doesn't affect the status of this allocated task. + */ + public void updatePersistentStatus(Task.Status status, PersistentTasksService.PersistentTaskOperationListener listener) { + persistentTasksService.updateStatus(persistentTaskId, status, listener); + } + + public long getPersistentTaskId() { + return persistentTaskId; + } + + void init(PersistentTasksService persistentTasksService, long persistentTaskId) { + this.persistentTasksService = persistentTasksService; + this.persistentTaskId = persistentTaskId; + } + + public Exception getFailure() { + return failure; + } + + boolean startNotification(Exception failure) { + boolean result = state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.FAILED); + if (result) { + this.failure = failure; + } + return result; + } + + boolean notificationFailed() { + return state.compareAndSet(AllocatedPersistentTask.State.FAILED, AllocatedPersistentTask.State.FAILED_NOTIFICATION); + } + + boolean restartCompletionNotification() { + return state.compareAndSet(AllocatedPersistentTask.State.FAILED_NOTIFICATION, AllocatedPersistentTask.State.FAILED); + } + + boolean markAsNotified() { + return state.compareAndSet(AllocatedPersistentTask.State.FAILED, AllocatedPersistentTask.State.NOTIFIED); + } + + boolean markAsCancelled() { + return state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.CANCELLED); + } + + public State getState() { + return state.get(); + } + + public enum State { + STARTED, // the task is currently running + CANCELLED, // the task is cancelled + FAILED, // the task is done running and trying to notify caller + FAILED_NOTIFICATION, // the caller notification failed + NOTIFIED // the caller was notified, the task can be removed + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTask.java deleted file mode 100644 index 6721775125caf..0000000000000 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTask.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.persistent; - -import org.elasticsearch.common.inject.Provider; -import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.tasks.TaskId; - -/** - * Represents a executor node operation that corresponds to a persistent task - */ -public class NodePersistentTask extends CancellableTask { - private Provider statusProvider; - - private long persistentTaskId; - - public NodePersistentTask(long id, String type, String action, String description, TaskId parentTask) { - super(id, type, action, description, parentTask); - } - - @Override - public boolean shouldCancelChildrenOnCancellation() { - return true; - } - - // In case of persistent tasks we always need to return: `false` - // because in case of persistent task the parent task isn't a task in the task manager, but in cluster state. - // This instructs the task manager not to try to kill this persistent task when the task manager cannot find - // a fake parent node id "cluster" in the cluster state - @Override - public final boolean cancelOnParentLeaving() { - return false; - } - - @Override - public Status getStatus() { - Provider statusProvider = this.statusProvider; - if (statusProvider != null) { - return statusProvider.get(); - } else { - return null; - } - } - - public void setStatusProvider(Provider statusProvider) { - assert this.statusProvider == null; - this.statusProvider = statusProvider; - } - - public long getPersistentTaskId() { - return persistentTaskId; - } - - public void setPersistentTaskId(long persistentTaskId) { - this.persistentTaskId = persistentTaskId; - } - -} diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index 3ad0bc57572e2..10dd76927b170 100644 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -34,7 +34,7 @@ public NodePersistentTasksExecutor(ThreadPool threadPool) { } public void executeTask(Request request, - NodePersistentTask task, + AllocatedPersistentTask task, PersistentTasksExecutor action, ActionListener listener) { threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java index b4c192ceab4c6..b07bfbfc79df5 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java @@ -30,6 +30,6 @@ public abstract class PersistentTaskRequest extends ActionRequest implements NamedWriteable, ToXContent { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId) { - return new NodePersistentTask(id, type, action, getDescription(), parentTaskId); + return new AllocatedPersistentTask(id, type, action, getDescription(), parentTaskId); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 862bd15c56d9b..31d97ce2a8271 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -107,7 +107,7 @@ public void validate(Request request, ClusterState clusterState) { * The status can be used to store the current progress of the task or provide an insight for the * task allocator about the state of the currently running tasks. */ - protected void updatePersistentTaskStatus(NodePersistentTask task, Task.Status status, ActionListener listener) { + protected void updatePersistentTaskStatus(AllocatedPersistentTask task, Task.Status status, ActionListener listener) { persistentTasksService.updateStatus(task.getPersistentTaskId(), status, new PersistentTaskOperationListener() { @Override @@ -129,7 +129,7 @@ public void onFailure(Exception e) { * possibly on a different node. If listener.onResponse() is called, the task is considered to be successfully * completed and will be removed from the cluster state and not restarted. */ - protected abstract void nodeOperation(NodePersistentTask task, Request request, ActionListener listener); + protected abstract void nodeOperation(AllocatedPersistentTask task, Request request, ActionListener listener); public String getExecutor() { return executor; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index 22fdca1b869ca..a8c1171dd4baa 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -24,10 +24,8 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Provider; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -47,7 +45,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import static java.util.Objects.requireNonNull; @@ -56,7 +53,7 @@ * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. */ public class PersistentTasksNodeService extends AbstractComponent implements ClusterStateListener { - private final Map runningTasks = new HashMap<>(); + private final Map runningTasks = new HashMap<>(); private final PersistentTasksService persistentTasksService; private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry; private final TaskManager taskManager; @@ -90,14 +87,14 @@ public void clusterChanged(ClusterChangedEvent event) { for (PersistentTask taskInProgress : tasks.tasks()) { if (localNodeId.equals(taskInProgress.getExecutorNode())) { PersistentTaskId persistentTaskId = new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()); - RunningPersistentTask persistentTask = runningTasks.get(persistentTaskId); + AllocatedPersistentTask persistentTask = runningTasks.get(persistentTaskId); if (persistentTask == null) { // New task - let's start it startTask(taskInProgress); } else { // The task is still running notVisitedTasks.remove(persistentTaskId); - if (persistentTask.getState() == State.FAILED_NOTIFICATION) { + if (persistentTask.getState() == AllocatedPersistentTask.State.FAILED_NOTIFICATION) { // We tried to notify the master about this task before but the notification failed and // the master doesn't seem to know about it - retry notification restartCompletionNotification(persistentTask); @@ -108,14 +105,14 @@ public void clusterChanged(ClusterChangedEvent event) { } for (PersistentTaskId id : notVisitedTasks) { - RunningPersistentTask task = runningTasks.get(id); - if (task.getState() == State.NOTIFIED || task.getState() == State.FAILED) { + AllocatedPersistentTask task = runningTasks.get(id); + if (task.getState() == AllocatedPersistentTask.State.NOTIFIED || task.getState() == AllocatedPersistentTask.State.FAILED) { // Result was sent to the caller and the caller acknowledged acceptance of the result finishTask(id); - } else if (task.getState() == State.FAILED_NOTIFICATION) { + } else if (task.getState() == AllocatedPersistentTask.State.FAILED_NOTIFICATION) { // We tried to send result to master, but it failed and master doesn't know about this task // this shouldn't really happen, unless this node is severally out of sync with the master - logger.warn("failed to notify master about task {}", task.getId()); + logger.warn("failed to notify master about task {}", task.getPersistentTaskId()); finishTask(id); } else { // task is running locally, but master doesn't know about it - that means that the persistent task was removed @@ -130,16 +127,14 @@ public void clusterChanged(ClusterChangedEvent event) { private void startTask(PersistentTask taskInProgress) { PersistentTasksExecutor action = persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName()); - NodePersistentTask task = (NodePersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]", + AllocatedPersistentTask task = (AllocatedPersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]", taskInProgress.getRequest()); boolean processed = false; try { - RunningPersistentTask runningPersistentTask = new RunningPersistentTask(task, taskInProgress.getId()); - task.setStatusProvider(runningPersistentTask); - task.setPersistentTaskId(taskInProgress.getId()); - PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask); + task.init(persistentTasksService, taskInProgress.getId()); + PersistentTaskListener listener = new PersistentTaskListener(task); try { - runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask); + runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), task); nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action, listener); } catch (Exception e) { // Submit task failure @@ -155,17 +150,17 @@ private void startTask(PersistentTask) () -> new ParameterizedMessage("failed to cancel task {}", task.getId()), e); + logger.warn((Supplier) () -> new ParameterizedMessage("failed to cancel task {}", task.getPersistentTaskId()), e); } }); } @@ -183,10 +178,10 @@ public void onFailure(Exception e) { } - private void restartCompletionNotification(RunningPersistentTask task) { - logger.trace("resending notification for task {}", task.getId()); - if (task.getState() == State.CANCELLED) { - taskManager.unregister(task.getTask()); + private void restartCompletionNotification(AllocatedPersistentTask task) { + logger.trace("resending notification for task {}", task.getPersistentTaskId()); + if (task.getState() == AllocatedPersistentTask.State.CANCELLED) { + taskManager.unregister(task); } else { if (task.restartCompletionNotification()) { // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier. @@ -201,35 +196,35 @@ public void onFailure(Exception e) { @Override protected void doRun() throws Exception { - persistentTasksService.sendCompletionNotification(task.getId(), task.getFailure(), listener); + persistentTasksService.sendCompletionNotification(task.getPersistentTaskId(), task.getFailure(), listener); } }); } catch (Exception e) { listener.onFailure(e); } } else { - logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState()); + logger.warn("attempt to resend notification for task {} in the {} state", task.getPersistentTaskId(), task.getState()); } } } - private void startCompletionNotification(RunningPersistentTask task, Exception e) { - if (task.getState() == State.CANCELLED) { - taskManager.unregister(task.getTask()); + private void startCompletionNotification(AllocatedPersistentTask task, Exception e) { + if (task.getState() == AllocatedPersistentTask.State.CANCELLED) { + taskManager.unregister(task); } else { - logger.trace("sending notification for failed task {}", task.getId()); + logger.trace("sending notification for failed task {}", task.getPersistentTaskId()); if (task.startNotification(e)) { - persistentTasksService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); + persistentTasksService.sendCompletionNotification(task.getPersistentTaskId(), e, new PublishedResponseListener(task)); } else { - logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState()); + logger.warn("attempt to send notification for task {} in the {} state", task.getPersistentTaskId(), task.getState()); } } } private class PersistentTaskListener implements ActionListener { - private final RunningPersistentTask task; + private final AllocatedPersistentTask task; - PersistentTaskListener(final RunningPersistentTask task) { + PersistentTaskListener(final AllocatedPersistentTask task) { this.task = task; } @@ -240,14 +235,14 @@ public void onResponse(Empty response) { @Override public void onFailure(Exception e) { - if (task.getTask().isCancelled()) { + if (task.isCancelled()) { // The task was explicitly cancelled - no need to restart it, just log the exception if it's not TaskCancelledException if (e instanceof TaskCancelledException == false) { logger.warn((Supplier) () -> new ParameterizedMessage( "cancelled task {} failed with an exception, cancellation reason [{}]", - task.getId(), task.getTask().getReasonCancelled()), e); + task.getPersistentTaskId(), task.getReasonCancelled()), e); } - if (CancelTasksRequest.DEFAULT_REASON.equals(task.getTask().getReasonCancelled())) { + if (CancelTasksRequest.DEFAULT_REASON.equals(task.getReasonCancelled())) { startCompletionNotification(task, null); } else { startCompletionNotification(task, e); @@ -259,39 +254,31 @@ public void onFailure(Exception e) { } private class PublishedResponseListener implements PersistentTaskOperationListener { - private final RunningPersistentTask task; + private final AllocatedPersistentTask task; - PublishedResponseListener(final RunningPersistentTask task) { + PublishedResponseListener(final AllocatedPersistentTask task) { this.task = task; } @Override public void onResponse(long taskId) { - logger.trace("notification for task {} was successful", task.getId()); + logger.trace("notification for task {} was successful", task.getPersistentTaskId()); if (task.markAsNotified() == false) { - logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getId(), task.getState()); + logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getPersistentTaskId(), task.getState()); } - taskManager.unregister(task.getTask()); + taskManager.unregister(task); } @Override public void onFailure(Exception e) { - logger.warn((Supplier) () -> new ParameterizedMessage("notification for task {} failed - retrying", task.getId()), e); + logger.warn((Supplier) () -> new ParameterizedMessage("notification for task {} failed - retrying", task.getPersistentTaskId()), e); if (task.notificationFailed() == false) { - logger.warn("attempt to mark restart notification for task {} in the {} state failed", task.getId(), task.getState()); + logger.warn("attempt to mark restart notification for task {} in the {} state failed", task.getPersistentTaskId(), task.getState()); } } } - public enum State { - STARTED, // the task is currently running - CANCELLED, // the task is cancelled - FAILED, // the task is done running and trying to notify caller - FAILED_NOTIFICATION, // the caller notification failed - NOTIFIED // the caller was notified, the task can be removed - } - private static class PersistentTaskId { private final long id; private final long allocationId; @@ -317,80 +304,17 @@ public int hashCode() { } } - private static class RunningPersistentTask implements Provider { - private final NodePersistentTask task; - private final long id; - private final AtomicReference state; - @Nullable - private Exception failure; - - RunningPersistentTask(NodePersistentTask task, long id) { - this(task, id, State.STARTED); - } - - RunningPersistentTask(NodePersistentTask task, long id, State state) { - this.task = task; - this.id = id; - this.state = new AtomicReference<>(state); - } - - public NodePersistentTask getTask() { - return task; - } - - public long getId() { - return id; - } - - public State getState() { - return state.get(); - } - - public Exception getFailure() { - return failure; - } - - public boolean startNotification(Exception failure) { - boolean result = state.compareAndSet(State.STARTED, State.FAILED); - if (result) { - this.failure = failure; - } - return result; - } - - public boolean notificationFailed() { - return state.compareAndSet(State.FAILED, State.FAILED_NOTIFICATION); - } - - public boolean restartCompletionNotification() { - return state.compareAndSet(State.FAILED_NOTIFICATION, State.FAILED); - } - - public boolean markAsNotified() { - return state.compareAndSet(State.FAILED, State.NOTIFIED); - } - - public boolean markAsCancelled() { - return state.compareAndSet(State.STARTED, State.CANCELLED); - } - - @Override - public Task.Status get() { - return new Status(state.get()); - } - } - public static class Status implements Task.Status { public static final String NAME = "persistent_executor"; - private final State state; + private final AllocatedPersistentTask.State state; - public Status(State state) { + public Status(AllocatedPersistentTask.State state) { this.state = requireNonNull(state, "State cannot be null"); } public Status(StreamInput in) throws IOException { - state = State.valueOf(in.readString()); + state = AllocatedPersistentTask.State.valueOf(in.readString()); } @Override @@ -416,7 +340,7 @@ public String toString() { return Strings.toString(this); } - public State getState() { + public AllocatedPersistentTask.State getState() { return state; } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java index 9d831b5563560..37925b7144fa5 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.persistent.PersistentTasksNodeService.State; import org.elasticsearch.persistent.PersistentTasksNodeService.Status; import static org.hamcrest.Matchers.containsString; @@ -29,7 +28,7 @@ public class PersistentTasksNodeServiceStatusTests extends AbstractWireSerializi @Override protected Status createTestInstance() { - return new Status(randomFrom(State.values())); + return new Status(randomFrom(AllocatedPersistentTask.State.values())); } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index c5dba05a12f62..1eb235ef639d2 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -338,11 +338,11 @@ private ClusterState removeTask(ClusterState state, long taskId) { private class Execution { private final PersistentTaskRequest request; - private final NodePersistentTask task; + private final AllocatedPersistentTask task; private final PersistentTasksExecutor holder; private final ActionListener listener; - Execution(PersistentTaskRequest request, NodePersistentTask task, PersistentTasksExecutor holder, + Execution(PersistentTaskRequest request, AllocatedPersistentTask task, PersistentTasksExecutor holder, ActionListener listener) { this.request = request; this.task = task; @@ -359,7 +359,7 @@ private class MockExecutor extends NodePersistentTasksExecutor { } @Override - public void executeTask(Request request, NodePersistentTask task, + public void executeTask(Request request, AllocatedPersistentTask task, PersistentTasksExecutor action, ActionListener listener) { executions.add(new Execution(request, task, action, listener)); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 040dadaf1ac93..57f3c7a502e78 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -340,7 +340,7 @@ public Assignment getAssignment(TestRequest request, ClusterState clusterState) } @Override - protected void nodeOperation(NodePersistentTask task, TestRequest request, ActionListener listener) { + protected void nodeOperation(AllocatedPersistentTask task, TestRequest request, ActionListener listener) { logger.info("started node operation for the task {}", task); try { TestTask testTask = (TestTask) task; @@ -423,7 +423,7 @@ public TestTasksRequestBuilder newRequestBuilder(ElasticsearchClient client) { } - public static class TestTask extends NodePersistentTask { + public static class TestTask extends AllocatedPersistentTask { private volatile String operation; public TestTask(long id, String type, String action, String description, TaskId parentTask) { From b142d7e29c5fc96f21f3fac31a7cf585ebfcf0a5 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 29 Mar 2017 12:54:44 -0400 Subject: [PATCH 15/50] Persistent Tasks: Remove unused stopped and removeOnCompletion flags (#853) The stopped and removeOnCompletion flags are not currently used, this commit removes them for now to simplify things. --- .../CreatePersistentTaskAction.java | 48 +------ .../PersistentTasksClusterService.java | 18 +-- .../PersistentTasksCustomMetaData.java | 69 +++------- .../persistent/PersistentTasksService.java | 13 -- .../PersistentTasksClusterServiceTests.java | 119 ++++++++---------- .../PersistentTasksCustomMetaDataTests.java | 14 +-- .../PersistentTasksExecutorFullRestartIT.java | 47 ++----- .../persistent/PersistentTasksExecutorIT.java | 61 --------- .../PersistentTasksNodeServiceTests.java | 6 +- 9 files changed, 84 insertions(+), 311 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java index d0d52ed4a7030..519bc21acc57f 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java @@ -69,10 +69,6 @@ public static class Request extends MasterNodeRequest { private PersistentTaskRequest request; - private boolean stopped; - - private boolean removeOnCompletion = true; - public Request() { } @@ -80,8 +76,6 @@ public Request() { public Request(String action, PersistentTaskRequest request) { this.action = action; this.request = request; - this.stopped = false; - this.removeOnCompletion = true; } @Override @@ -89,8 +83,6 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); action = in.readString(); request = in.readNamedWriteable(PersistentTaskRequest.class); - stopped = in.readBoolean(); - removeOnCompletion = in.readBoolean(); } @Override @@ -98,8 +90,6 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(action); out.writeNamedWriteable(request); - out.writeBoolean(stopped); - out.writeBoolean(removeOnCompletion); } @Override @@ -120,14 +110,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Request request1 = (Request) o; return Objects.equals(action, request1.action) && - Objects.equals(request, request1.request) && - removeOnCompletion == request1.removeOnCompletion && - stopped == request1.stopped; + Objects.equals(request, request1.request); } @Override public int hashCode() { - return Objects.hash(action, request, removeOnCompletion, stopped); + return Objects.hash(action, request); } public String getAction() { @@ -146,21 +134,6 @@ public void setRequest(PersistentTaskRequest request) { this.request = request; } - public boolean isStopped() { - return stopped; - } - - public void setStopped(boolean stopped) { - this.stopped = stopped; - } - - public boolean shouldRemoveOnCompletion() { - return removeOnCompletion; - } - - public void setRemoveOnCompletion(boolean removeOnCompletion) { - this.removeOnCompletion = removeOnCompletion; - } } public static class RequestBuilder extends MasterNodeOperationRequestBuilder { @@ -235,7 +193,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.createPersistentTask(request.action, request.request, request.stopped, request.removeOnCompletion, + persistentTasksClusterService.createPersistentTask(request.action, request.request, new ActionListener() { @Override public void onResponse(Long newTaskId) { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 8203ff3396c88..5f22dd25a22d4 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -61,21 +61,15 @@ public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorR * @param request request * @param listener the listener that will be called when task is started */ - public void createPersistentTask(String action, Request request, boolean stopped, - boolean removeOnCompletion, + public void createPersistentTask(String action, Request request, ActionListener listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { validate(action, clusterService.state(), request); final Assignment assignment; - if (stopped) { - // the task is stopped no need to assign it anywhere - assignment = PersistentTasksCustomMetaData.FINISHED_TASK_ASSIGNMENT; - } else { - assignment = getAssignement(action, currentState, request); - } - return update(currentState, builder(currentState).addTask(action, request, stopped, removeOnCompletion, assignment)); + assignment = getAssignement(action, currentState, request); + return update(currentState, builder(currentState).addTask(action, request, assignment)); } @Override @@ -319,11 +313,7 @@ static ClusterState reassignTasks(ClusterState currentState, Logger logger, Exec logger.trace("ignoring task {} because assignment is the same {}", task.getId(), assignment); } } else { - if (task.isStopped()) { - logger.trace("ignoring task {} because it is stopped", task.getId()); - } else { - logger.trace("ignoring task {} because it is still running", task.getId()); - } + logger.trace("ignoring task {} because it is still running", task.getId()); } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 3805a510c394e..a5048052a18da 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -97,8 +97,6 @@ public PersistentTasksCustomMetaData(long currentId, Map PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setId, new ParseField("id")); PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setTaskName, new ParseField("name")); PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id")); - PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setRemoveOnCompletion, new ParseField("remove_on_completion")); - PERSISTENT_TASK_PARSER.declareBoolean(TaskBuilder::setStopped, new ParseField("stopped")); PERSISTENT_TASK_PARSER.declareNamedObjects( (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { @@ -240,8 +238,6 @@ public static class PersistentTask implem private final long allocationId; private final String taskName; private final Request request; - private final boolean stopped; - private final boolean removeOnCompletion; @Nullable private final Status status; private final Assignment assignment; @@ -249,29 +245,27 @@ public static class PersistentTask implem private final Long allocationIdOnLastStatusUpdate; - public PersistentTask(long id, String taskName, Request request, boolean stopped, boolean removeOnCompletion, Assignment assignment) { - this(id, 0L, taskName, request, stopped, removeOnCompletion, null, assignment, null); + public PersistentTask(long id, String taskName, Request request, Assignment assignment) { + this(id, 0L, taskName, request, null, assignment, null); } - public PersistentTask(PersistentTask task, boolean stopped, Assignment assignment) { - this(task.id, task.allocationId + 1L, task.taskName, task.request, stopped, task.removeOnCompletion, task.status, + public PersistentTask(PersistentTask task, Assignment assignment) { + this(task.id, task.allocationId + 1L, task.taskName, task.request, task.status, assignment, task.allocationId); } public PersistentTask(PersistentTask task, Status status) { - this(task.id, task.allocationId, task.taskName, task.request, task.stopped, task.removeOnCompletion, status, + this(task.id, task.allocationId, task.taskName, task.request, status, task.assignment, task.allocationId); } - private PersistentTask(long id, long allocationId, String taskName, Request request, boolean stopped, boolean removeOnCompletion, + private PersistentTask(long id, long allocationId, String taskName, Request request, Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; this.taskName = taskName; this.request = request; this.status = status; - this.stopped = stopped; - this.removeOnCompletion = removeOnCompletion; this.assignment = assignment; this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; // Update parent request for starting tasks with correct parent task ID @@ -284,8 +278,6 @@ private PersistentTask(StreamInput in) throws IOException { allocationId = in.readLong(); taskName = in.readString(); request = (Request) in.readNamedWriteable(PersistentTaskRequest.class); - stopped = in.readBoolean(); - removeOnCompletion = in.readBoolean(); status = in.readOptionalNamedWriteable(Task.Status.class); assignment = new Assignment(in.readOptionalString(), in.readString()); allocationIdOnLastStatusUpdate = in.readOptionalLong(); @@ -297,8 +289,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(allocationId); out.writeString(taskName); out.writeNamedWriteable(request); - out.writeBoolean(stopped); - out.writeBoolean(removeOnCompletion); out.writeOptionalNamedWriteable(status); out.writeOptionalString(assignment.executorNode); out.writeString(assignment.explanation); @@ -314,8 +304,6 @@ public boolean equals(Object o) { allocationId == that.allocationId && Objects.equals(taskName, that.taskName) && Objects.equals(request, that.request) && - stopped == that.stopped && - removeOnCompletion == that.removeOnCompletion && Objects.equals(status, that.status) && Objects.equals(assignment, that.assignment) && Objects.equals(allocationIdOnLastStatusUpdate, that.allocationIdOnLastStatusUpdate); @@ -323,7 +311,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(id, allocationId, taskName, request, stopped, removeOnCompletion, status, assignment, + return Objects.hash(id, allocationId, taskName, request, status, assignment, allocationIdOnLastStatusUpdate); } @@ -365,7 +353,7 @@ public boolean isAssigned() { * Returns true if the tasks is not stopped and unassigned or assigned to a non-existing node. */ public boolean needsReassignment(DiscoveryNodes nodes) { - return isStopped() == false && (assignment.isAssigned() == false || nodes.nodeExists(assignment.getExecutorNode()) == false); + return (assignment.isAssigned() == false || nodes.nodeExists(assignment.getExecutorNode()) == false); } @Nullable @@ -373,14 +361,6 @@ public Status getStatus() { return status; } - public boolean isStopped() { - return stopped; - } - - public boolean shouldRemoveOnCompletion() { - return removeOnCompletion; - } - /** * @return Whether the task status isn't stale. When a task gets unassigned from the executor node or assigned * to a new executor node and the status hasn't been updated then the task status is stale. @@ -421,8 +401,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("allocation_id_on_last_status_update", allocationIdOnLastStatusUpdate); } } - builder.field("stopped", stopped); - builder.field("remove_on_completion", removeOnCompletion); } builder.endObject(); return builder; @@ -439,8 +417,6 @@ private static class TaskBuilder { private long allocationId; private String taskName; private Request request; - private boolean stopped = true; - private boolean removeOnCompletion; private Status status; private Assignment assignment = INITIAL_ASSIGNMENT; private Long allocationIdOnLastStatusUpdate; @@ -471,16 +447,6 @@ public TaskBuilder setStatus(Status status) { } - public TaskBuilder setStopped(boolean stopped) { - this.stopped = stopped; - return this; - } - - public TaskBuilder setRemoveOnCompletion(boolean removeOnCompletion) { - this.removeOnCompletion = removeOnCompletion; - return this; - } - public TaskBuilder setAssignment(Assignment assignment) { this.assignment = assignment; return this; @@ -492,7 +458,7 @@ public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdO } public PersistentTask build() { - return new PersistentTask<>(id, allocationId, taskName, request, stopped, removeOnCompletion, status, + return new PersistentTask<>(id, allocationId, taskName, request, status, assignment, allocationIdOnLastStatusUpdate); } } @@ -578,11 +544,10 @@ private Builder setTasks(List * After the task is added its id can be found by calling {{@link #getCurrentId()}} method. */ - public Builder addTask(String taskName, Request request, boolean stopped, - boolean removeOnCompletion, Assignment assignment) { + public Builder addTask(String taskName, Request request, Assignment assignment) { changed = true; currentId++; - tasks.put(currentId, new PersistentTask<>(currentId, taskName, request, stopped, removeOnCompletion, assignment)); + tasks.put(currentId, new PersistentTask<>(currentId, taskName, request, assignment)); return this; } @@ -593,7 +558,7 @@ public Builder reassignTask(long taskId, Assignment assignment) { PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; - tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); + tasks.put(taskId, new PersistentTask<>(taskInProgress, assignment)); } return this; } @@ -609,9 +574,9 @@ public Builder assignTask(long taskId, PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request); - if (assignment.isAssigned() || taskInProgress.isStopped()) { + if (assignment.isAssigned()) { changed = true; - tasks.put(taskId, new PersistentTask<>(taskInProgress, false, assignment)); + tasks.put(taskId, new PersistentTask<>(taskInProgress, assignment)); } } return this; @@ -649,11 +614,7 @@ public Builder finishTask(long taskId) { PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; - if (taskInProgress.removeOnCompletion) { - tasks.remove(taskId); - } else { - tasks.put(taskId, new PersistentTask<>(taskInProgress, true, FINISHED_TASK_ASSIGNMENT)); - } + tasks.remove(taskId); } return this; } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 807b4d63860d3..ff3e49c8d799b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -42,27 +42,14 @@ public PersistentTasksService(Settings settings, ClusterService clusterService, this.clusterService = clusterService; } - /** - * Creates the specified persistent action and tries to start it immediately, upon completion the task is - * removed from the cluster state - */ - public void createPersistentActionTask(String action, Request request, - PersistentTaskOperationListener listener) { - createPersistentActionTask(action, request, false, true, listener); - } - /** * Creates the specified persistent action. The action is started unless the stopped parameter is equal to true. * If removeOnCompletion parameter is equal to true, the task is removed from the cluster state upon completion. * Otherwise it will remain there in the stopped state. */ public void createPersistentActionTask(String action, Request request, - boolean stopped, - boolean removeOnCompletion, PersistentTaskOperationListener listener) { CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(action, request); - createPersistentActionRequest.setStopped(stopped); - createPersistentActionRequest.setRemoveOnCompletion(removeOnCompletion); try { client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( o -> listener.onResponse(o.getTaskId()), listener::onFailure)); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index fd538f8d6ec01..e5dda15996152 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -92,7 +92,7 @@ public void testReassignConsidersClusterStateUpdates() { addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(2, 40); for (int i = 0; i < numberOfTasks; i++) { - addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits", false); + addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits"); } MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); @@ -113,21 +113,17 @@ public void testReassignTasks() { addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(0, 40); for (int i = 0; i < numberOfTasks; i++) { - switch (randomInt(3)) { + switch (randomInt(2)) { case 0: // add an unassigned task that should get assigned because it's assigned to a non-existing node or unassigned - addTask(tasks, "should_assign", "assign_me", randomBoolean() ? null : "no_longer_exits", false); + addTask(tasks, "should_assign", "assign_me", randomBoolean() ? null : "no_longer_exits"); break; case 1: // add a task assigned to non-existing node that should not get assigned - addTask(tasks, "should_not_assign", "dont_assign_me", randomBoolean() ? null : "no_longer_exits", false); + addTask(tasks, "should_not_assign", "dont_assign_me", randomBoolean() ? null : "no_longer_exits"); break; case 2: - // add a stopped task assigned to non-existing node that should not get assigned - addTask(tasks, "should_not_assign", "fail_me_if_called", null, true); - break; - case 3: - addTask(tasks, "assign_one", "assign_one", randomBoolean() ? null : "no_longer_exits", false); + addTask(tasks, "assign_one", "assign_one", randomBoolean() ? null : "no_longer_exits"); break; } @@ -145,39 +141,34 @@ public void testReassignTasks() { int assignOneCount = 0; for (PersistentTask task : tasksInProgress.tasks()) { - if (task.isStopped()) { - assertThat("stopped tasks should be never assigned", task.getExecutorNode(), nullValue()); - assertThat(task.getAssignment().getExplanation(), equalTo("explanation: " + task.getTaskName())); - } else { - // explanation should correspond to the action name - switch (task.getTaskName()) { - case "should_assign": - assertThat(task.getExecutorNode(), notNullValue()); - assertThat(task.isAssigned(), equalTo(true)); - if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { - logger.info(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE).toString()); - } - assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), - clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); + // explanation should correspond to the action name + switch (task.getTaskName()) { + case "should_assign": + assertThat(task.getExecutorNode(), notNullValue()); + assertThat(task.isAssigned(), equalTo(true)); + if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { + logger.info(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE).toString()); + } + assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), + clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); + assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + break; + case "should_not_assign": + assertThat(task.getExecutorNode(), nullValue()); + assertThat(task.isAssigned(), equalTo(false)); + assertThat(task.getAssignment().getExplanation(), equalTo("no appropriate nodes found for the assignment")); + break; + case "assign_one": + if (task.isAssigned()) { + assignOneCount++; + assertThat("more than one assign_one tasks are assigned", assignOneCount, lessThanOrEqualTo(1)); assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); - break; - case "should_not_assign": - assertThat(task.getExecutorNode(), nullValue()); - assertThat(task.isAssigned(), equalTo(false)); - assertThat(task.getAssignment().getExplanation(), equalTo("no appropriate nodes found for the assignment")); - break; - case "assign_one": - if (task.isAssigned()) { - assignOneCount++; - assertThat("more than one assign_one tasks are assigned", assignOneCount, lessThanOrEqualTo(1)); - assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); - } else { - assertThat(task.getAssignment().getExplanation(), equalTo("only one task can be assigned at a time")); - } - break; - default: - fail("Unknown action " + task.getTaskName()); - } + } else { + assertThat(task.getAssignment().getExplanation(), equalTo("only one task can be assigned at a time")); + } + break; + default: + fail("Unknown action " + task.getTaskName()); } } } @@ -218,8 +209,7 @@ public Assignment getAssignment( private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { DiscoveryNodes nodes = clusterState.nodes(); PersistentTasksCustomMetaData tasksInProgress = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - if (tasksInProgress.findTasks("assign_one", - task -> task.isStopped() == false && nodes.nodeExists(task.getExecutorNode())).isEmpty()) { + if (tasksInProgress.findTasks("assign_one", task -> nodes.nodeExists(task.getExecutorNode())).isEmpty()) { return randomNodeAssignment(clusterState.nodes()); } else { return new Assignment(null, "only one task can be assigned at a time"); @@ -269,13 +259,12 @@ private ClusterState significantChange(ClusterState clusterState) { // we don't have any unassigned tasks - add some if (randomBoolean()) { logger.info("added random task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), null, - false); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), null); tasksOrNodesChanged = true; } else { logger.info("added unassignable task with custom assignment message"); addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), - new Assignment(null, "change me"), "never_assign", false); + new Assignment(null, "change me"), "never_assign"); tasksOrNodesChanged = true; } } @@ -332,23 +321,16 @@ private ClusterState insignificantChange(ClusterState clusterState) { } if (randomBoolean()) { logger.info("added random unassignable task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), tasksBuilder, NO_NODE_FOUND, "never_assign", false); - return builder.build(); - } - if (randomBoolean()) { - // add unassigned task in stopped state - logger.info("added random stopped task"); - addRandomTask(builder, MetaData.builder(clusterState.metaData()), tasksBuilder, null, true); - return builder.build(); - } else { - logger.info("changed routing table"); - MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); - metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); - RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); - changeRoutingTable(metaData, routingTable); - builder.metaData(metaData).routingTable(routingTable.build()); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), tasksBuilder, NO_NODE_FOUND, "never_assign"); return builder.build(); } + logger.info("changed routing table"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); + RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); + changeRoutingTable(metaData, routingTable); + builder.metaData(metaData).routingTable(routingTable.build()); + return builder.build(); } } if (randomBoolean()) { @@ -398,9 +380,6 @@ private boolean hasAssignableTasks(PersistentTasksCustomMetaData tasks, Discover return false; } return tasks.tasks().stream().anyMatch(task -> { - if (task.isStopped()) { - return false; - } if (task.getExecutorNode() == null || discoveryNodes.nodeExists(task.getExecutorNode())) { return "never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) == false; } @@ -415,20 +394,20 @@ private boolean hasTasksAssignedTo(PersistentTasksCustomMetaData tasks, String n private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, - String node, boolean stopped) { + String node) { return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)), - randomAsciiOfLength(10), stopped); + randomAsciiOfLength(10)); } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, - Assignment assignment, String param, boolean stopped) { + Assignment assignment, String param) { return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, - tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), stopped, randomBoolean(), assignment).build())); + tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), assignment).build())); } - private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node, boolean stopped) { - tasks.addTask(action, new TestRequest(param), stopped, randomBoolean(), new Assignment(node, "explanation: " + action)); + private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node) { + tasks.addTask(action, new TestRequest(param), new Assignment(node, "explanation: " + action)); } private DiscoveryNode newNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index dfeb7152e1cf5..3465407db5204 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -57,9 +57,8 @@ protected PersistentTasksCustomMetaData createTestInstance() { int numberOfTasks = randomInt(10); PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); for (int i = 0; i < numberOfTasks; i++) { - boolean stopped = randomBoolean(); tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), - stopped, randomBoolean(), stopped ? new Assignment(null, "stopped") : randomAssignment()); + randomAssignment()); if (randomBoolean()) { // From time to time update status tasks.updateTaskStatus(tasks.getCurrentId(), new Status(randomAsciiOfLength(10))); @@ -149,9 +148,7 @@ protected XContentBuilder toXContent(Custom instance, XContentType contentType, } private Builder addRandomTask(Builder builder) { - boolean stopped = randomBoolean(); - builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), stopped, randomBoolean(), - stopped ? new Assignment(null, "stopped") : randomAssignment()); + builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), randomAssignment()); return builder; } @@ -196,7 +193,6 @@ public void testSerializationContext() throws Exception { assertEquals(testTask.getId(), newTask.getId()); assertEquals(testTask.getStatus(), newTask.getStatus()); assertEquals(testTask.getRequest(), newTask.getRequest()); - assertEquals(testTask.isStopped(), newTask.isStopped()); // Things that shouldn't be serialized assertEquals(0, newTask.getAllocationId()); @@ -233,16 +229,12 @@ public void testBuilder() { if (randomBoolean()) { // Trying to reassign to the same node builder.assignTask(lastKnownTask, (s, request) -> task.getAssignment()); - // should change if the task was stopped AND unassigned - if (task.getExecutorNode() == null && task.isStopped()) { - changed = true; - } } else { // Trying to reassign to a different node Assignment randomAssignment = randomAssignment(); builder.assignTask(lastKnownTask, (s, request) -> randomAssignment); // should change if the task was unassigned and was reassigned to a different node or started - if ((task.isAssigned() == false && randomAssignment.isAssigned()) || task.isStopped()) { + if ((task.isAssigned() == false && randomAssignment.isAssigned())) { changed = true; } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index a13d683673fe5..a4c7061da050c 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -60,35 +60,26 @@ public void testFullClusterRestart() throws Exception { long[] taskIds = new long[numberOfTasks]; List futures = new ArrayList<>(numberOfTasks); - boolean[] stopped = new boolean[numberOfTasks]; - int runningTasks = 0; for (int i = 0; i < numberOfTasks; i++) { - stopped[i] = randomBoolean(); - if (stopped[i] == false) { - runningTasks++; - } PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); futures.add(future); - service.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped[i], true, future); + service.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); } for (int i = 0; i < numberOfTasks; i++) { taskIds[i] = futures.get(i).get(); } - final int numberOfRunningTasks = runningTasks; PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() .custom(PersistentTasksCustomMetaData.TYPE); assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); - if (numberOfRunningTasks > 0) { - // Make sure that at least one of the tasks is running - assertBusy(() -> { - // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() - .getTasks().size(), greaterThan(0)); - }); - } + // Make sure that at least one of the tasks is running + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), greaterThan(0)); + }); // Restart cluster internalCluster().fullRestart(); @@ -100,30 +91,6 @@ public void testFullClusterRestart() throws Exception { for (int i = 0; i < numberOfTasks; i++) { PersistentTask task = tasksInProgress.getTask(taskIds[i]); assertNotNull(task); - assertThat(task.isStopped(), equalTo(stopped[i])); - } - - logger.info("Waiting for {} original tasks to start", numberOfRunningTasks); - assertBusy(() -> { - // Wait for the running task to start automatically - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() - .getTasks().size(), equalTo(numberOfRunningTasks)); - }); - - // Start all other tasks - tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - service = internalCluster().getInstance(PersistentTasksService.class); - for (int i = 0; i < numberOfTasks; i++) { - PersistentTask task = tasksInProgress.getTask(taskIds[i]); - assertNotNull(task); - logger.info("checking task with id {} stopped {} node {}", task.getId(), task.isStopped(), task.getExecutorNode()); - assertThat(task.isStopped(), equalTo(stopped[i])); - assertThat(task.getExecutorNode(), stopped[i] ? nullValue() : notNullValue()); - if (stopped[i]) { - PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture(); - service.startTask(task.getId(), startFuture); - assertEquals(startFuture.get(), (Long) task.getId()); - } } logger.info("Waiting for {} tasks to start", numberOfTasks); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 87793860caa1f..6bc3d297be482 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -126,67 +126,6 @@ public void testPersistentActionCompletion() throws Exception { stopOrCancelTask(firstRunningTask.getTaskId()); } - public void testPersistentActionCompletionWithoutRemoval() throws Exception { - boolean stopped = randomBoolean(); - PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); - persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), stopped, false, - future); - long taskId = future.get(); - - PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksCustomMetaData.TYPE); - assertThat(tasksInProgress.tasks().size(), equalTo(1)); - assertThat(tasksInProgress.getTask(taskId).isStopped(), equalTo(stopped)); - assertThat(tasksInProgress.getTask(taskId).getExecutorNode(), stopped ? nullValue() : notNullValue()); - assertThat(tasksInProgress.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); - - int numberOfIters = randomIntBetween(1, 5); // we will start/stop the action a few times before removing it - logger.info("start/stop the task {} times stating with stopped {}", numberOfIters, stopped); - for (int i = 0; i < numberOfIters; i++) { - logger.info("iteration {}", i); - if (stopped) { - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() - .getTasks(), empty()); - PersistentTaskOperationFuture startFuture = new PersistentTaskOperationFuture(); - persistentTasksService.startTask(taskId, startFuture); - assertEquals(startFuture.get(), (Long) taskId); - } - assertBusy(() -> { - // Wait for the task to start - assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() - .getTasks().size(), equalTo(1)); - }); - TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") - .get().getTasks().get(0); - - stopOrCancelTask(firstRunningTask.getTaskId()); - - assertBusy(() -> { - // Wait for the task to finish - List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") - .get().getTasks(); - logger.info("Found {} tasks", tasks.size()); - assertThat(tasks.size(), equalTo(0)); - }); - stopped = true; - } - - assertBusy(() -> { - // Wait for the task to be marked as stopped - PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksCustomMetaData.TYPE); - assertThat(tasks.tasks().size(), equalTo(1)); - assertThat(tasks.getTask(taskId).isStopped(), equalTo(true)); - assertThat(tasks.getTask(taskId).shouldRemoveOnCompletion(), equalTo(false)); - }); - - logger.info("Removing action record from cluster state"); - PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); - persistentTasksService.removeTask(taskId, removeFuture); - assertEquals(removeFuture.get(), (Long) taskId); - } - public void testPersistentActionWithNoAvailableNode() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 1eb235ef639d2..5f74f6963c722 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -90,11 +90,11 @@ public void testStartTask() throws Exception { boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { - tasks.addTask("test_action", new TestRequest("other_" + i), false, true, + tasks.addTask("test_action", new TestRequest("other_" + i), new Assignment("other_node_" + randomInt(nonLocalNodesCount), "test assignment on other node")); if (added == false && randomBoolean()) { added = true; - tasks.addTask("test", new TestRequest("this_param"), false, true, + tasks.addTask("test", new TestRequest("this_param"), new Assignment("this_node", "test assignment on this node")); } } @@ -317,7 +317,7 @@ private ClusterState addTask(ClusterStat PersistentTasksCustomMetaData.Builder builder = PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, - builder.addTask(action, request, false, true, new Assignment(node, "test assignment")).build())).build(); + builder.addTask(action, request, new Assignment(node, "test assignment")).build())).build(); } private ClusterState reallocateTask(ClusterState state, long taskId, String node) { From 78b844e79bddd052d25d15f4548584fb426e4eb8 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 28 Mar 2017 14:55:50 +0200 Subject: [PATCH 16/50] Check allocationIdOnLastStatusUpdate when trying to detect whether a task is stale. --- .../elasticsearch/persistent/PersistentTasksCustomMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index a5048052a18da..a0f6fa3d9f7ad 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -366,7 +366,7 @@ public Status getStatus() { * to a new executor node and the status hasn't been updated then the task status is stale. */ public boolean isCurrentStatus() { - return allocationIdOnLastStatusUpdate == allocationId; + return allocationIdOnLastStatusUpdate != null && allocationIdOnLastStatusUpdate == allocationId; } @Override From 6ca044736e82db5ff30a2f0e86788a4f814562ed Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 31 Mar 2017 16:05:34 -0400 Subject: [PATCH 17/50] Persistent Tasks: Add waitForPersistentTaskStatus method (#901) This method allows to wait for tasks to change their status to match the supplied predicate. --- .../PersistentTasksCustomMetaData.java | 12 ++++- .../persistent/PersistentTasksService.java | 47 +++++++++++++++---- .../persistent/PersistentTasksExecutorIT.java | 37 +++++++++------ .../PersistentTasksNodeServiceTests.java | 4 +- .../persistent/TestPersistentTasksPlugin.java | 2 +- 5 files changed, 74 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index a0f6fa3d9f7ad..a101da93e5a62 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -20,6 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -182,6 +183,15 @@ public static PersistentTasksCustomMetaData fromXContent(XContentParser parser) return PERSISTENT_TASKS_PARSER.parse(parser, null).build(); } + @SuppressWarnings("unchecked") + public static PersistentTask getTaskWithId(ClusterState clusterState, long taskId) { + PersistentTasksCustomMetaData tasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + if (tasks != null) { + return (PersistentTask)tasks.getTask(taskId); + } + return null; + } + public static class Assignment { @Nullable private final String executorNode; @@ -228,8 +238,6 @@ public String toString() { public static final Assignment INITIAL_ASSIGNMENT = new Assignment(null, "waiting for initial assignment"); - public static final Assignment FINISHED_TASK_ASSIGNMENT = new Assignment(null, "task has finished"); - /** * A record that represents a single running persistent task */ diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index ff3e49c8d799b..4a23096e8afa9 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -21,12 +21,21 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.util.function.Predicate; /** * This service is used by persistent actions to propagate changes in the action state and notify about completion @@ -35,11 +44,13 @@ public class PersistentTasksService extends AbstractComponent { private final Client client; private final ClusterService clusterService; + private final ThreadPool threadPool; - public PersistentTasksService(Settings settings, ClusterService clusterService, Client client) { + public PersistentTasksService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { super(settings); this.client = client; this.clusterService = clusterService; + this.threadPool = threadPool; } /** @@ -115,15 +126,33 @@ public void removeTask(long taskId, PersistentTaskOperationListener listener) { } /** - * Starts a persistent task + * Waits for the persistent task with giving id (taskId) to achieve the desired status. */ - public void startTask(long taskId, PersistentTaskOperationListener listener) { - StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(taskId); - try { - client.execute(StartPersistentTaskAction.INSTANCE, startRequest, ActionListener.wrap(o -> listener.onResponse(taskId), - listener::onFailure)); - } catch (Exception e) { - listener.onFailure(e); + public void waitForPersistentTaskStatus(long taskId, Predicate> predicate, @Nullable TimeValue timeout, + WaitForPersistentTaskStatusListener listener) { + ClusterStateObserver stateObserver = new ClusterStateObserver(clusterService, timeout, logger, threadPool.getThreadContext()); + stateObserver.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + listener.onResponse(taskId); + } + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + + } + + @Override + public void onTimeout(TimeValue timeout) { + listener.onTimeout(timeout); + } + }, clusterState -> predicate.test(PersistentTasksCustomMetaData.getTaskWithId(clusterState, taskId))); + } + + public interface WaitForPersistentTaskStatusListener extends PersistentTaskOperationListener { + default void onTimeout(TimeValue timeout) { + onFailure(new IllegalStateException("timed out after " + timeout)); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 6bc3d297be482..3b9eeb68ed53e 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -20,12 +20,13 @@ package org.elasticsearch.persistent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.BaseFuture; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.persistent.PersistentTasksService.WaitForPersistentTaskStatusListener; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder; @@ -34,12 +35,10 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2) @@ -64,7 +63,7 @@ public void cleanup() throws Exception { assertNoRunningTasks(); } - public static class PersistentTaskOperationFuture extends BaseFuture implements PersistentTaskOperationListener { + public static class PersistentTaskOperationFuture extends BaseFuture implements WaitForPersistentTaskStatusListener { @Override public void onResponse(long taskId) { @@ -166,7 +165,7 @@ public void testPersistentActionStatusUpdate() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - future.get(); + long taskId = future.get(); assertBusy(() -> { // Wait for the task to start @@ -189,20 +188,30 @@ public void testPersistentActionStatusUpdate() throws Exception { .get().getTasks().size(), equalTo(1)); int finalI = i; - assertBusy(() -> { - PersistentTasksCustomMetaData tasks = internalCluster().clusterService().state().getMetaData() - .custom(PersistentTasksCustomMetaData.TYPE); - assertThat(tasks.tasks().size(), equalTo(1)); - assertThat(tasks.tasks().iterator().next().getStatus(), notNullValue()); - assertThat(tasks.tasks().iterator().next().getStatus().toString(), equalTo("{\"phase\":\"phase " + (finalI + 1) + "\"}")); - }); - + PersistentTaskOperationFuture future1 = new PersistentTaskOperationFuture(); + persistentTasksService.waitForPersistentTaskStatus(taskId, + task -> task != null && task.isCurrentStatus()&& task.getStatus().toString() != null && + task.getStatus().toString().equals("{\"phase\":\"phase " + (finalI + 1) + "\"}"), + TimeValue.timeValueSeconds(10), future1); + assertThat(future1.get(), equalTo(taskId)); } + PersistentTaskOperationFuture future1 = new PersistentTaskOperationFuture(); + persistentTasksService.waitForPersistentTaskStatus(taskId, + task -> false, TimeValue.timeValueMillis(10), future1); + + expectThrows(Exception.class, future1::get); + + // Wait for the task to disappear + PersistentTaskOperationFuture future2 = new PersistentTaskOperationFuture(); + persistentTasksService.waitForPersistentTaskStatus(taskId, Objects::isNull, TimeValue.timeValueSeconds(10), future2); + logger.info("Completing the running task"); // Complete the running task and make sure it finishes properly assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) .get().getTasks().size(), equalTo(1)); + + assertThat(future2.get(), equalTo(taskId)); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 5f74f6963c722..576b7c6172216 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -164,7 +164,7 @@ public void testTaskCancellation() { ClusterService clusterService = createClusterService(); AtomicLong capturedTaskId = new AtomicLong(); AtomicReference capturedListener = new AtomicReference<>(); - PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null) { + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null, null) { @Override public void sendCancellation(long taskId, PersistentTaskOperationListener listener) { capturedTaskId.set(taskId); @@ -242,7 +242,7 @@ public void testNotificationFailure() throws Exception { AtomicReference capturedException = new AtomicReference<>(); AtomicReference capturedListener = new AtomicReference<>(); PersistentTasksService persistentTasksService = - new PersistentTasksService(Settings.EMPTY, clusterService, null) { + new PersistentTasksService(Settings.EMPTY, clusterService, null, null) { @Override public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) { capturedTaskId.set(taskId); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 57f3c7a502e78..7bbba5c916c4f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -103,7 +103,7 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry) { - PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, client); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, client); TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, persistentTasksService, clusterService); PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, From 1b0f5b95720b513315fea030b332235d31dc4505 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sat, 1 Apr 2017 18:17:07 -0400 Subject: [PATCH 18/50] Persistent Tasks: require correct allocation id for status updates (#923) In order to prevent tasks state updates by stale executors, this commit adds a check for correct allocation id during status update operation. --- .../persistent/AllocatedPersistentTask.java | 6 ++++-- .../NodePersistentTasksExecutor.java | 2 ++ .../PersistentTasksClusterService.java | 12 ++++++++--- .../PersistentTasksCustomMetaData.java | 11 ++++++++++ .../persistent/PersistentTasksExecutor.java | 21 ------------------- .../PersistentTasksNodeService.java | 2 +- .../persistent/PersistentTasksService.java | 10 ++++++--- .../UpdatePersistentTaskStatusAction.java | 18 +++++++++++----- .../persistent/PersistentTasksExecutorIT.java | 20 ++++++++++-------- .../persistent/TestPersistentTasksPlugin.java | 5 +++-- .../UpdatePersistentTaskRequestTests.java | 2 +- 11 files changed, 62 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index 6bad551f98494..4caffda83d65a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -32,6 +32,7 @@ */ public class AllocatedPersistentTask extends CancellableTask { private long persistentTaskId; + private long allocationId; private final AtomicReference state; @Nullable @@ -70,16 +71,17 @@ public Status getStatus() { * This doesn't affect the status of this allocated task. */ public void updatePersistentStatus(Task.Status status, PersistentTasksService.PersistentTaskOperationListener listener) { - persistentTasksService.updateStatus(persistentTaskId, status, listener); + persistentTasksService.updateStatus(persistentTaskId, allocationId, status, listener); } public long getPersistentTaskId() { return persistentTaskId; } - void init(PersistentTasksService persistentTasksService, long persistentTaskId) { + void init(PersistentTasksService persistentTasksService, long persistentTaskId, long allocationId) { this.persistentTasksService = persistentTasksService; this.persistentTaskId = persistentTaskId; + this.allocationId = allocationId; } public Exception getFailure() { diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index 10dd76927b170..d96540b6c9dfa 100644 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -25,6 +25,8 @@ /** * This component is responsible for execution of persistent tasks. + * + * It abstracts away the execution of tasks and greatly simplifies testing of PersistentTasksNodeService */ public class NodePersistentTasksExecutor { private final ThreadPool threadPool; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 5f22dd25a22d4..1cdcd190ee7eb 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -192,18 +192,24 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * Update task status * * @param id the id of a persistent task + * @param allocationId the expected allocation id of the persistent task * @param status new status * @param listener the listener that will be called when task is removed */ - public void updatePersistentTaskStatus(long id, Task.Status status, ActionListener listener) { + public void updatePersistentTaskStatus(long id, long allocationId, Task.Status status, ActionListener listener) { clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); - if (tasksInProgress.hasTask(id)) { + if (tasksInProgress.hasTask(id, allocationId)) { return update(currentState, tasksInProgress.updateTaskStatus(id, status)); } else { - throw new ResourceNotFoundException("the task with id {} doesn't exist", id); + if (tasksInProgress.hasTask(id)) { + logger.warn("trying to update status on task {} with unexpected allocation id {}", id, allocationId); + } else { + logger.warn("trying to update status on non-existing task {}", id); + } + throw new ResourceNotFoundException("the task with id {} and allocation id {} doesn't exist", id, allocationId); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index a101da93e5a62..2dd475f1a9a4a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -634,6 +634,17 @@ public boolean hasTask(long taskId) { return tasks.containsKey(taskId); } + /** + * Checks if the task is currently present in the list and has the right allocation id + */ + public boolean hasTask(long taskId, long allocationId) { + PersistentTask taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + return taskInProgress.getAllocationId() == allocationId; + } + return false; + } + /** * Returns the id of the last added task */ diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 31d97ce2a8271..c7c92a77a66b9 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -101,27 +101,6 @@ public void validate(Request request, ClusterState clusterState) { } - /** - * Updates the persistent task status in the cluster state. - *

- * The status can be used to store the current progress of the task or provide an insight for the - * task allocator about the state of the currently running tasks. - */ - protected void updatePersistentTaskStatus(AllocatedPersistentTask task, Task.Status status, ActionListener listener) { - persistentTasksService.updateStatus(task.getPersistentTaskId(), status, - new PersistentTaskOperationListener() { - @Override - public void onResponse(long taskId) { - listener.onResponse(Empty.INSTANCE); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - } - /** * This operation will be executed on the executor node. *

diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index a8c1171dd4baa..9302db379c38c 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -131,7 +131,7 @@ private void startTask(PersistentTask listener.onResponse(taskId), listener::onFailure)); diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java index d15b5ac8b54cf..9bb87ec23045b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -68,15 +68,16 @@ public Response newResponse() { public static class Request extends MasterNodeRequest { private long taskId; - + private long allocationId; private Task.Status status; public Request() { } - public Request(long taskId, Task.Status status) { + public Request(long taskId, long allocationId, Task.Status status) { this.taskId = taskId; + this.allocationId = allocationId; this.status = status; } @@ -84,6 +85,10 @@ public void setTaskId(long taskId) { this.taskId = taskId; } + public void setAllocationId(long allocationId) { + this.allocationId = allocationId; + } + public void setStatus(Task.Status status) { this.status = status; } @@ -92,6 +97,7 @@ public void setStatus(Task.Status status) { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); taskId = in.readLong(); + allocationId = in.readLong(); status = in.readOptionalNamedWriteable(Task.Status.class); } @@ -99,6 +105,7 @@ public void readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeLong(taskId); + out.writeLong(allocationId); out.writeOptionalNamedWriteable(status); } @@ -112,13 +119,13 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return taskId == request.taskId && + return taskId == request.taskId && allocationId == request.allocationId && Objects.equals(status, request.status); } @Override public int hashCode() { - return Objects.hash(taskId, status); + return Objects.hash(taskId, allocationId, status); } } @@ -207,7 +214,8 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener() { + persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.allocationId, request.status, + new ActionListener() { @Override public void onResponse(Empty empty) { listener.onResponse(new Response(true)); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 3b9eeb68ed53e..7c6f7411624f4 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -19,14 +19,16 @@ package org.elasticsearch.persistent; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.BaseFuture; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.persistent.PersistentTasksService.WaitForPersistentTaskStatusListener; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder; @@ -37,6 +39,7 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -63,17 +66,11 @@ public void cleanup() throws Exception { assertNoRunningTasks(); } - public static class PersistentTaskOperationFuture extends BaseFuture implements WaitForPersistentTaskStatusListener { - + public static class PersistentTaskOperationFuture extends PlainActionFuture implements WaitForPersistentTaskStatusListener { @Override public void onResponse(long taskId) { set(taskId); } - - @Override - public void onFailure(Exception e) { - setException(e); - } } public void testPersistentActionFailure() throws Exception { @@ -200,7 +197,12 @@ public void testPersistentActionStatusUpdate() throws Exception { persistentTasksService.waitForPersistentTaskStatus(taskId, task -> false, TimeValue.timeValueMillis(10), future1); - expectThrows(Exception.class, future1::get); + assertThrows(future1, IllegalStateException.class, "timed out after 10ms"); + + PersistentTaskOperationFuture failedUpdateFuture = new PersistentTaskOperationFuture(); + persistentTasksService.updateStatus(taskId, -1, new Status("should fail"), failedUpdateFuture); + assertThrows(failedUpdateFuture, ResourceNotFoundException.class, "the task with id " + taskId + + " and allocation id -1 doesn't exist"); // Wait for the task to disappear PersistentTaskOperationFuture future2 = new PersistentTaskOperationFuture(); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 7bbba5c916c4f..1c126e2c32c2f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -63,6 +63,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; import java.io.IOException; import java.util.ArrayList; @@ -365,9 +366,9 @@ protected void nodeOperation(AllocatedPersistentTask task, TestRequest request, CountDownLatch latch = new CountDownLatch(1); Status status = new Status("phase " + phase.incrementAndGet()); logger.info("updating the task status to {}", status); - updatePersistentTaskStatus(task, status, new ActionListener() { + task.updatePersistentStatus(status, new PersistentTaskOperationListener() { @Override - public void onResponse(Empty empty) { + public void onResponse(long taskId) { logger.info("updating was successful"); latch.countDown(); } diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java index 3cf9e8bbc6245..94f69e13923db 100644 --- a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java @@ -30,7 +30,7 @@ public class UpdatePersistentTaskRequestTests extends AbstractStreamableTestCase @Override protected Request createTestInstance() { - return new Request(randomLong(), new Status(randomAsciiOfLength(10))); + return new Request(randomLong(), randomLong(), new Status(randomAsciiOfLength(10))); } @Override From a5acb556b08499319fdc7b27f65cb94dcc6aa5b3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 3 Apr 2017 10:02:46 +0200 Subject: [PATCH 19/50] Use PersistentTasksService#waitForPersistentTaskStatus(...) to wait for job and datafeed status and use PersistentTasksService#removeTask(...) to force close job and force stop datafeed. --- .../elasticsearch/persistent/PersistentTasksService.java | 6 +++--- .../elasticsearch/persistent/TestPersistentTasksPlugin.java | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index c583f266cdec5..f17677a6e2db3 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -20,7 +20,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -34,6 +33,7 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.security.InternalClient; import java.util.function.Predicate; @@ -42,11 +42,11 @@ */ public class PersistentTasksService extends AbstractComponent { - private final Client client; + private final InternalClient client; private final ClusterService clusterService; private final ThreadPool threadPool; - public PersistentTasksService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { + public PersistentTasksService(Settings settings, ClusterService clusterService, ThreadPool threadPool, InternalClient client) { super(settings); this.client = client; this.clusterService = clusterService; diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 1c126e2c32c2f..62288ed38e35d 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -64,6 +64,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.security.InternalClient; import java.io.IOException; import java.util.ArrayList; @@ -104,7 +105,8 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry) { - PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, client); + InternalClient internalClient = new InternalClient(Settings.EMPTY, threadPool, client); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, internalClient); TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, persistentTasksService, clusterService); PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, From 5b45b167bdf9c3041638c2d3502a3cf61530a348 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 4 Apr 2017 09:44:56 -0400 Subject: [PATCH 20/50] Persistent Tasks: check the current state in waitForPersistentTaskStatus (#935) Add a check for the current state waitForPersistentTaskStatus before waiting for the next one. This fixes sporadic failure in testPersistentActionStatusUpdate test. Fixes #928 --- .../persistent/PersistentTasksService.java | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index f17677a6e2db3..717ee6da6863b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -130,28 +130,32 @@ public void removeTask(long taskId, PersistentTaskOperationListener listener) { } /** - * Waits for the persistent task with giving id (taskId) to achieve the desired status. + * Checks if the persistent task with giving id (taskId) has the desired state and if it doesn't + * waits of it. */ public void waitForPersistentTaskStatus(long taskId, Predicate> predicate, @Nullable TimeValue timeout, WaitForPersistentTaskStatusListener listener) { ClusterStateObserver stateObserver = new ClusterStateObserver(clusterService, timeout, logger, threadPool.getThreadContext()); - stateObserver.waitForNextChange(new ClusterStateObserver.Listener() { - @Override - public void onNewClusterState(ClusterState state) { - listener.onResponse(taskId); - } - - @Override - public void onClusterServiceClose() { - listener.onFailure(new NodeClosedException(clusterService.localNode())); - - } - - @Override - public void onTimeout(TimeValue timeout) { - listener.onTimeout(timeout); - } - }, clusterState -> predicate.test(PersistentTasksCustomMetaData.getTaskWithId(clusterState, taskId))); + if (predicate.test(PersistentTasksCustomMetaData.getTaskWithId(stateObserver.setAndGetObservedState(), taskId))) { + listener.onResponse(taskId); + } else { + stateObserver.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + listener.onResponse(taskId); + } + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + listener.onTimeout(timeout); + } + }, clusterState -> predicate.test(PersistentTasksCustomMetaData.getTaskWithId(clusterState, taskId))); + } } public interface WaitForPersistentTaskStatusListener extends PersistentTaskOperationListener { From 97822dbea386c783582230583ed7618d4afb31a4 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 4 Apr 2017 11:04:37 -0400 Subject: [PATCH 21/50] Respond to rename random ASCII helper methods This commit is response to the renaming of the random ASCII helper methods in ESTestCase. The name of this method was changed because these methods only produce random strings generated from [a-zA-Z], not from all ASCII characters. --- .../PersistentTasksClusterServiceTests.java | 14 +++++++------- .../PersistentTasksCustomMetaDataTests.java | 14 +++++++------- .../StartPersistentActionRequestTests.java | 8 ++++---- .../UpdatePersistentTaskRequestTests.java | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index e5dda15996152..aea9849ba58dd 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -271,7 +271,7 @@ private ClusterState significantChange(ClusterState clusterState) { // add a node if there are unassigned tasks if (clusterState.nodes().getNodes().isEmpty()) { logger.info("added random node"); - builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAsciiOfLength(10)))); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAlphaOfLength(10)))); tasksOrNodesChanged = true; } @@ -317,7 +317,7 @@ private ClusterState insignificantChange(ClusterState clusterState) { // we don't have any unassigned tasks - adding a node or changing a routing table shouldn't affect anything if (randomBoolean()) { logger.info("added random node"); - builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAsciiOfLength(10)))); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAlphaOfLength(10)))); } if (randomBoolean()) { logger.info("added random unassignable task"); @@ -365,7 +365,7 @@ private ClusterState insignificantChange(ClusterState clusterState) { } } // Just add a random index - that shouldn't change anything - IndexMetaData indexMetaData = IndexMetaData.builder(randomAsciiOfLength(10)) + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) .numberOfShards(1) .numberOfReplicas(1) @@ -395,15 +395,15 @@ private boolean hasTasksAssignedTo(PersistentTasksCustomMetaData tasks, String n private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, String node) { - return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAsciiOfLength(10)), - randomAsciiOfLength(10)); + return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAlphaOfLength(10)), + randomAlphaOfLength(10)); } private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, Assignment assignment, String param) { return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, - tasks.addTask(randomAsciiOfLength(10), new TestRequest(param), assignment).build())); + tasks.addTask(randomAlphaOfLength(10), new TestRequest(param), assignment).build())); } private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node) { @@ -437,7 +437,7 @@ private ClusterState initialState() { } private void changeRoutingTable(MetaData.Builder metaData, RoutingTable.Builder routingTable) { - IndexMetaData indexMetaData = IndexMetaData.builder(randomAsciiOfLength(10)) + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) .numberOfShards(1) .numberOfReplicas(1) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index 3465407db5204..d20042ba601a0 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -57,11 +57,11 @@ protected PersistentTasksCustomMetaData createTestInstance() { int numberOfTasks = randomInt(10); PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); for (int i = 0; i < numberOfTasks; i++) { - tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), + tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), randomAssignment()); if (randomBoolean()) { // From time to time update status - tasks.updateTaskStatus(tasks.getCurrentId(), new Status(randomAsciiOfLength(10))); + tasks.updateTaskStatus(tasks.getCurrentId(), new Status(randomAlphaOfLength(10))); } } return tasks.build(); @@ -101,7 +101,7 @@ protected Custom makeTestChanges(Custom testInstance) { if (tasksInProgress.tasks().isEmpty()) { addRandomTask(builder); } else { - builder.updateTaskStatus(pickRandomTask(tasksInProgress), randomBoolean() ? new Status(randomAsciiOfLength(10)) : null); + builder.updateTaskStatus(pickRandomTask(tasksInProgress), randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); } break; case 3: @@ -148,7 +148,7 @@ protected XContentBuilder toXContent(Custom instance, XContentType contentType, } private Builder addRandomTask(Builder builder) { - builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAsciiOfLength(10)), randomAssignment()); + builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), randomAssignment()); return builder; } @@ -247,7 +247,7 @@ public void testBuilder() { if (builder.hasTask(lastKnownTask)) { changed = true; } - builder.updateTaskStatus(lastKnownTask, randomBoolean() ? new Status(randomAsciiOfLength(10)) : null); + builder.updateTaskStatus(lastKnownTask, randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); break; case 4: if (builder.hasTask(lastKnownTask)) { @@ -274,9 +274,9 @@ private Assignment randomAssignment() { if (randomBoolean()) { return NO_NODE_FOUND; } else { - return new Assignment(null, randomAsciiOfLength(10)); + return new Assignment(null, randomAlphaOfLength(10)); } } - return new Assignment(randomAsciiOfLength(10), randomAsciiOfLength(10)); + return new Assignment(randomAlphaOfLength(10), randomAlphaOfLength(10)); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java index 52b3c245ef490..fa425454a0993 100644 --- a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -33,15 +33,15 @@ public class StartPersistentActionRequestTests extends AbstractStreamableTestCas protected Request createTestInstance() { TestRequest testRequest = new TestRequest(); if (randomBoolean()) { - testRequest.setTestParam(randomAsciiOfLengthBetween(1, 20)); + testRequest.setTestParam(randomAlphaOfLengthBetween(1, 20)); } if (randomBoolean()) { - testRequest.setParentTask(randomAsciiOfLengthBetween(1, 20), randomLong()); + testRequest.setParentTask(randomAlphaOfLengthBetween(1, 20), randomLong()); } if (randomBoolean()) { - testRequest.setExecutorNodeAttr(randomAsciiOfLengthBetween(1, 20)); + testRequest.setExecutorNodeAttr(randomAlphaOfLengthBetween(1, 20)); } - return new Request(randomAsciiOfLengthBetween(1, 20), new TestRequest()); + return new Request(randomAlphaOfLengthBetween(1, 20), new TestRequest()); } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java index 94f69e13923db..bc80c0d7211f3 100644 --- a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java @@ -30,7 +30,7 @@ public class UpdatePersistentTaskRequestTests extends AbstractStreamableTestCase @Override protected Request createTestInstance() { - return new Request(randomLong(), randomLong(), new Status(randomAsciiOfLength(10))); + return new Request(randomLong(), randomLong(), new Status(randomAlphaOfLength(10))); } @Override From 5a8512bf4e28cfc24d220a266f2aaae6da7ab956 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 4 Apr 2017 13:56:22 -0400 Subject: [PATCH 22/50] Persistent Tasks: refactor PersistentTasksService to use ActionListener (#937) PersistentTasksService methods are not using ActionListener> instead of PersistentTaskOperationListener. --- .../persistent/AllocatedPersistentTask.java | 2 +- .../CompletionPersistentTaskAction.java | 77 ++----- .../CreatePersistentTaskAction.java | 8 +- .../persistent/PersistentTaskResponse.java | 19 +- .../PersistentTasksClusterService.java | 56 ++--- .../PersistentTasksCustomMetaData.java | 2 +- .../persistent/PersistentTasksExecutor.java | 7 +- .../PersistentTasksNodeService.java | 14 +- .../persistent/PersistentTasksService.java | 53 ++--- .../RemovePersistentTaskAction.java | 58 ++--- .../persistent/StartPersistentTaskAction.java | 212 ------------------ .../UpdatePersistentTaskStatusAction.java | 58 ++--- .../CancelPersistentTaskResponseTests.java | 35 --- .../PersistentTasksExecutorFullRestartIT.java | 10 +- .../persistent/PersistentTasksExecutorIT.java | 61 +++-- .../PersistentTasksExecutorResponseTests.java | 22 +- .../PersistentTasksNodeServiceTests.java | 17 +- .../persistent/TestPersistentTasksPlugin.java | 15 +- 18 files changed, 184 insertions(+), 542 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java delete mode 100644 server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index 4caffda83d65a..b2115e5dd95ea 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -70,7 +70,7 @@ public Status getStatus() { * * This doesn't affect the status of this allocated task. */ - public void updatePersistentStatus(Task.Status status, PersistentTasksService.PersistentTaskOperationListener listener) { + public void updatePersistentStatus(Task.Status status, ActionListener> listener) { persistentTasksService.updateStatus(persistentTaskId, allocationId, status, listener); } diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java index 95b81cfdea45d..ed16de1cabf3b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -37,8 +36,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Objects; @@ -48,7 +47,7 @@ * removed from the cluster state in case of successful completion or restarted on some other node in case of failure. */ public class CompletionPersistentTaskAction extends Action { public static final CompletionPersistentTaskAction INSTANCE = new CompletionPersistentTaskAction(); @@ -64,8 +63,8 @@ public RequestBuilder newRequestBuilder(ElasticsearchClient client) { } @Override - public Response newResponse() { - return new Response(); + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } public static class Request extends MasterNodeRequest { @@ -117,49 +116,15 @@ public int hashCode() { } } - public static class Response extends AcknowledgedResponse { - public Response() { - super(); - } - - public Response(boolean acknowledged) { - super(acknowledged); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - readAcknowledged(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - writeAcknowledged(out); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AcknowledgedResponse that = (AcknowledgedResponse) o; - return isAcknowledged() == that.isAcknowledged(); - } - - @Override - public int hashCode() { - return Objects.hash(isAcknowledged()); - } - - } - public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + PersistentTaskResponse, CompletionPersistentTaskAction.RequestBuilder> { protected RequestBuilder(ElasticsearchClient client, CompletionPersistentTaskAction action) { super(client, action, new Request()); } } - public static class TransportAction extends TransportMasterNodeAction { + public static class TransportAction extends TransportMasterNodeAction { private final PersistentTasksClusterService persistentTasksClusterService; @@ -179,8 +144,8 @@ protected String executor() { } @Override - protected Response newResponse() { - return new Response(); + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } @Override @@ -190,18 +155,20 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) } @Override - protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.completePersistentTask(request.taskId, request.exception, new ActionListener() { - @Override - public void onResponse(Empty empty) { - listener.onResponse(newResponse()); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTasksClusterService.completePersistentTask(request.taskId, request.exception, + new ActionListener>() { + @Override + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java index 519bc21acc57f..892c51d41b86a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Objects; @@ -194,10 +195,11 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { persistentTasksClusterService.createPersistentTask(request.action, request.request, - new ActionListener() { + new ActionListener>() { + @Override - public void onResponse(Long newTaskId) { - listener.onResponse(new PersistentTaskResponse(newTaskId)); + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); } @Override diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java index 14ce2e3036522..4387ea4230fcc 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Objects; @@ -29,30 +30,30 @@ * Response upon a successful start or an persistent task */ public class PersistentTaskResponse extends ActionResponse { - private long taskId; + private PersistentTask task; public PersistentTaskResponse() { super(); } - public PersistentTaskResponse(long taskId) { - this.taskId = taskId; + public PersistentTaskResponse(PersistentTask task) { + this.task = task; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - taskId = in.readLong(); + task = in.readOptionalWriteable(PersistentTask::new); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeLong(taskId); + out.writeOptionalWriteable(task); } - public long getTaskId() { - return taskId; + public PersistentTask getTask() { + return task; } @Override @@ -60,11 +61,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; PersistentTaskResponse that = (PersistentTaskResponse) o; - return taskId == that.taskId; + return Objects.equals(task, that.task); } @Override public int hashCode() { - return Objects.hash(taskId); + return Objects.hash(task); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 1cdcd190ee7eb..6e391dc4a66e1 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -62,7 +62,7 @@ public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorR * @param listener the listener that will be called when task is started */ public void createPersistentTask(String action, Request request, - ActionListener listener) { + ActionListener> listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -77,10 +77,15 @@ public void onFailure(String source, Exception e) { listener.onFailure(e); } + @SuppressWarnings("unchecked") @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse( - ((PersistentTasksCustomMetaData) newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)).getCurrentId()); + PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + if (tasks != null) { + listener.onResponse(tasks.getTask(tasks.getCurrentId())); + } else { + listener.onResponse(null); + } } }); } @@ -93,7 +98,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @param failure the reason for restarting the task or null if the task completed successfully * @param listener the listener that will be called when task is removed */ - public void completePersistentTask(long id, Exception failure, ActionListener listener) { + public void completePersistentTask(long id, Exception failure, ActionListener> listener) { final String source; if (failure != null) { logger.warn("persistent task " + id + " failed", failure); @@ -122,38 +127,8 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(Empty.INSTANCE); - } - }); - } - - /** - * Switches the persistent task from stopped to started mode - * - * @param id the id of a persistent task - * @param listener the listener that will be called when task is removed - */ - public void startPersistentTask(long id, ActionListener listener) { - clusterService.submitStateUpdateTask("start persistent task", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); - if (tasksInProgress.hasTask(id)) { - return update(currentState, tasksInProgress - .assignTask(id, (action, request) -> getAssignement(action, currentState, request))); - } else { - throw new ResourceNotFoundException("the task with id {} doesn't exist", id); - } - } - - @Override - public void onFailure(String source, Exception e) { - listener.onFailure(e); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(Empty.INSTANCE); + // Using old state since in the new state the task is already gone + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(oldState, id)); } }); } @@ -164,7 +139,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @param id the id of a persistent task * @param listener the listener that will be called when task is removed */ - public void removePersistentTask(long id, ActionListener listener) { + public void removePersistentTask(long id, ActionListener> listener) { clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -183,7 +158,8 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(Empty.INSTANCE); + // Using old state since in the new state the task is already gone + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(oldState, id)); } }); } @@ -196,7 +172,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @param status new status * @param listener the listener that will be called when task is removed */ - public void updatePersistentTaskStatus(long id, long allocationId, Task.Status status, ActionListener listener) { + public void updatePersistentTaskStatus(long id, long allocationId, Task.Status status, ActionListener> listener) { clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -220,7 +196,7 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(Empty.INSTANCE); + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(newState, id)); } }); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 2dd475f1a9a4a..5366b432b95aa 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -281,7 +281,7 @@ private PersistentTask(long id, long allocationId, String taskName, Request requ } @SuppressWarnings("unchecked") - private PersistentTask(StreamInput in) throws IOException { + public PersistentTask(StreamInput in) throws IOException { id = in.readLong(); allocationId = in.readLong(); taskName = in.readString(); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index c7c92a77a66b9..656b83187589f 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -24,9 +24,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import java.util.function.Predicate; @@ -39,14 +37,11 @@ public abstract class PersistentTasksExecutor() { @Override - public void onResponse(long taskId) { - logger.trace("Persistent task with id {} was cancelled", taskId); + public void onResponse(CancelTasksResponse cancelTasksResponse) { + logger.trace("Persistent task with id {} was cancelled", task.getId()); } @@ -253,7 +253,7 @@ public void onFailure(Exception e) { } } - private class PublishedResponseListener implements PersistentTaskOperationListener { + private class PublishedResponseListener implements ActionListener> { private final AllocatedPersistentTask task; PublishedResponseListener(final AllocatedPersistentTask task) { @@ -262,8 +262,8 @@ private class PublishedResponseListener implements PersistentTaskOperationListen @Override - public void onResponse(long taskId) { - logger.trace("notification for task {} was successful", task.getPersistentTaskId()); + public void onResponse(PersistentTask persistentTask) { + logger.trace("notification for task {} was successful", task.getId()); if (task.markAsNotified() == false) { logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getPersistentTaskId(), task.getState()); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 717ee6da6863b..064b89794394c 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -54,16 +55,15 @@ public PersistentTasksService(Settings settings, ClusterService clusterService, } /** - * Creates the specified persistent action. The action is started unless the stopped parameter is equal to true. - * If removeOnCompletion parameter is equal to true, the task is removed from the cluster state upon completion. - * Otherwise it will remain there in the stopped state. + * Creates the specified persistent task and attempts to assign it to a node. */ - public void createPersistentActionTask(String action, Request request, - PersistentTaskOperationListener listener) { - CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(action, request); + @SuppressWarnings("unchecked") + public void startPersistentTask(String taskName, Request request, + ActionListener> listener) { + CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(taskName, request); try { client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( - o -> listener.onResponse(o.getTaskId()), listener::onFailure)); + o -> listener.onResponse((PersistentTask) o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); } @@ -71,13 +71,12 @@ public void createPersistentActionTask(S /** * Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure - * */ - public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) { + public void sendCompletionNotification(long taskId, Exception failure, ActionListener> listener) { CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); try { - client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, ActionListener.wrap(o -> listener.onResponse(taskId), - listener::onFailure)); + client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, + ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); } @@ -86,14 +85,13 @@ public void sendCompletionNotification(long taskId, Exception failure, Persisten /** * Cancels the persistent task. */ - public void sendCancellation(long taskId, PersistentTaskOperationListener listener) { + void sendCancellation(long taskId, ActionListener listener) { DiscoveryNode localNode = clusterService.localNode(); CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); cancelTasksRequest.setReason("persistent action was removed"); try { - client.admin().cluster().cancelTasks(cancelTasksRequest, ActionListener.wrap(o -> listener.onResponse(taskId), - listener::onFailure)); + client.admin().cluster().cancelTasks(cancelTasksRequest, listener); } catch (Exception e) { listener.onFailure(e); } @@ -101,28 +99,28 @@ public void sendCancellation(long taskId, PersistentTaskOperationListener listen /** * Updates status of the persistent task. - * + *

* Persistent task implementers shouldn't call this method directly and use * {@link AllocatedPersistentTask#updatePersistentStatus} instead */ - void updateStatus(long taskId, long allocationId, Task.Status status, PersistentTaskOperationListener listener) { + void updateStatus(long taskId, long allocationId, Task.Status status, ActionListener> listener) { UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, allocationId, status); try { client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, ActionListener.wrap( - o -> listener.onResponse(taskId), listener::onFailure)); + o -> listener.onResponse(o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); } } /** - * Removes a persistent task + * Cancels if needed and removes a persistent task */ - public void removeTask(long taskId, PersistentTaskOperationListener listener) { + public void cancelPersistentTask(long taskId, ActionListener> listener) { RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId); try { - client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(taskId), + client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); @@ -134,15 +132,15 @@ public void removeTask(long taskId, PersistentTaskOperationListener listener) { * waits of it. */ public void waitForPersistentTaskStatus(long taskId, Predicate> predicate, @Nullable TimeValue timeout, - WaitForPersistentTaskStatusListener listener) { + WaitForPersistentTaskStatusListener listener) { ClusterStateObserver stateObserver = new ClusterStateObserver(clusterService, timeout, logger, threadPool.getThreadContext()); if (predicate.test(PersistentTasksCustomMetaData.getTaskWithId(stateObserver.setAndGetObservedState(), taskId))) { - listener.onResponse(taskId); + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(stateObserver.setAndGetObservedState(), taskId)); } else { stateObserver.waitForNextChange(new ClusterStateObserver.Listener() { @Override public void onNewClusterState(ClusterState state) { - listener.onResponse(taskId); + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(state, taskId)); } @Override @@ -158,15 +156,10 @@ public void onTimeout(TimeValue timeout) { } } - public interface WaitForPersistentTaskStatusListener extends PersistentTaskOperationListener { + public interface WaitForPersistentTaskStatusListener + extends ActionListener> { default void onTimeout(TimeValue timeout) { onFailure(new IllegalStateException("timed out after " + timeout)); } } - - public interface PersistentTaskOperationListener { - void onResponse(long taskId); - void onFailure(Exception e); - } - } diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java index b4d636d258a96..0209d7745395d 100644 --- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -39,12 +39,13 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Objects; public class RemovePersistentTaskAction extends Action { public static final RemovePersistentTaskAction INSTANCE = new RemovePersistentTaskAction(); @@ -60,8 +61,8 @@ public RequestBuilder newRequestBuilder(ElasticsearchClient client) { } @Override - public Response newResponse() { - return new Response(); + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } public static class Request extends MasterNodeRequest { @@ -111,42 +112,8 @@ public int hashCode() { } } - public static class Response extends AcknowledgedResponse { - public Response() { - super(); - } - - public Response(boolean acknowledged) { - super(acknowledged); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - readAcknowledged(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - writeAcknowledged(out); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AcknowledgedResponse that = (AcknowledgedResponse) o; - return isAcknowledged() == that.isAcknowledged(); - } - - @Override - public int hashCode() { - return Objects.hash(isAcknowledged()); - } - - } - public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + PersistentTaskResponse, RemovePersistentTaskAction.RequestBuilder> { protected RequestBuilder(ElasticsearchClient client, RemovePersistentTaskAction action) { super(client, action, new Request()); @@ -159,7 +126,7 @@ public final RequestBuilder setTaskId(long taskId) { } - public static class TransportAction extends TransportMasterNodeAction { + public static class TransportAction extends TransportMasterNodeAction { private final PersistentTasksClusterService persistentTasksClusterService; @@ -179,8 +146,8 @@ protected String executor() { } @Override - protected Response newResponse() { - return new Response(); + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } @Override @@ -190,11 +157,12 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) } @Override - protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.removePersistentTask(request.taskId, new ActionListener() { + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTasksClusterService.removePersistentTask(request.taskId, new ActionListener>() { @Override - public void onResponse(Empty empty) { - listener.onResponse(new Response(true)); + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); } @Override diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java deleted file mode 100644 index 6621088c41484..0000000000000 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.persistent; - -import org.elasticsearch.action.Action; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; -import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.transport.TransportService; - -import java.io.IOException; -import java.util.Objects; - -/** - * This action can be used to start a persistent task previously created using {@link CreatePersistentTaskAction} - */ -public class StartPersistentTaskAction extends Action { - - public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction(); - public static final String NAME = "cluster:admin/persistent/start"; - - private StartPersistentTaskAction() { - super(NAME); - } - - @Override - public RequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new RequestBuilder(client, this); - } - - @Override - public Response newResponse() { - return new Response(); - } - - public static class Request extends MasterNodeRequest { - - private long taskId; - - public Request() { - - } - - public Request(long taskId) { - this.taskId = taskId; - } - - public void setTaskId(long taskId) { - this.taskId = taskId; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - taskId = in.readLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeLong(taskId); - } - - @Override - public ActionRequestValidationException validate() { - return null; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Request request = (Request) o; - return taskId == request.taskId; - } - - @Override - public int hashCode() { - return Objects.hash(taskId); - } - } - - public static class Response extends AcknowledgedResponse { - public Response() { - super(); - } - - public Response(boolean acknowledged) { - super(acknowledged); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - readAcknowledged(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - writeAcknowledged(out); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AcknowledgedResponse that = (AcknowledgedResponse) o; - return isAcknowledged() == that.isAcknowledged(); - } - - @Override - public int hashCode() { - return Objects.hash(isAcknowledged()); - } - - } - - public static class RequestBuilder extends MasterNodeOperationRequestBuilder { - - protected RequestBuilder(ElasticsearchClient client, StartPersistentTaskAction action) { - super(client, action, new Request()); - } - - public final RequestBuilder setTaskId(long taskId) { - request.setTaskId(taskId); - return this; - } - - } - - public static class TransportAction extends TransportMasterNodeAction { - - private final PersistentTasksClusterService persistentTasksClusterService; - - @Inject - public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, ActionFilters actionFilters, - PersistentTasksClusterService persistentTasksClusterService, - IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, - indexNameExpressionResolver, Request::new); - this.persistentTasksClusterService = persistentTasksClusterService; - } - - @Override - protected String executor() { - return ThreadPool.Names.MANAGEMENT; - } - - @Override - protected Response newResponse() { - return new Response(); - } - - @Override - protected ClusterBlockException checkBlock(Request request, ClusterState state) { - // Cluster is not affected but we look up repositories in metadata - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); - } - - @Override - protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.startPersistentTask(request.taskId, new ActionListener() { - @Override - public void onResponse(Empty empty) { - listener.onResponse(new Response(true)); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - } - } -} - - diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java index 9bb87ec23045b..772d200b24eb8 100644 --- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -40,12 +40,13 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Objects; public class UpdatePersistentTaskStatusAction extends Action { public static final UpdatePersistentTaskStatusAction INSTANCE = new UpdatePersistentTaskStatusAction(); @@ -61,8 +62,8 @@ public RequestBuilder newRequestBuilder(ElasticsearchClient client) { } @Override - public Response newResponse() { - return new Response(); + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } public static class Request extends MasterNodeRequest { @@ -129,42 +130,8 @@ public int hashCode() { } } - public static class Response extends AcknowledgedResponse { - public Response() { - super(); - } - - public Response(boolean acknowledged) { - super(acknowledged); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - readAcknowledged(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - writeAcknowledged(out); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AcknowledgedResponse that = (AcknowledgedResponse) o; - return isAcknowledged() == that.isAcknowledged(); - } - - @Override - public int hashCode() { - return Objects.hash(isAcknowledged()); - } - - } - public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + PersistentTaskResponse, UpdatePersistentTaskStatusAction.RequestBuilder> { protected RequestBuilder(ElasticsearchClient client, UpdatePersistentTaskStatusAction action) { super(client, action, new Request()); @@ -182,7 +149,7 @@ public final RequestBuilder setStatus(Task.Status status) { } - public static class TransportAction extends TransportMasterNodeAction { + public static class TransportAction extends TransportMasterNodeAction { private final PersistentTasksClusterService persistentTasksClusterService; @@ -202,8 +169,8 @@ protected String executor() { } @Override - protected Response newResponse() { - return new Response(); + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); } @Override @@ -213,12 +180,13 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) } @Override - protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.allocationId, request.status, - new ActionListener() { + new ActionListener>() { @Override - public void onResponse(Empty empty) { - listener.onResponse(new Response(true)); + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java deleted file mode 100644 index a2db4fad92fea..0000000000000 --- a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskResponseTests.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.persistent; - -import org.elasticsearch.persistent.RemovePersistentTaskAction.Response; -import org.elasticsearch.test.AbstractStreamableTestCase; - -public class CancelPersistentTaskResponseTests extends AbstractStreamableTestCase { - - @Override - protected Response createTestInstance() { - return new Response(randomBoolean()); - } - - @Override - protected Response createBlankInstance() { - return new Response(); - } -} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index a4c7061da050c..fc6b3abee7766 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -18,10 +18,10 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.persistent.PersistentTasksExecutorIT.PersistentTaskOperationFuture; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; @@ -58,16 +58,16 @@ public void testFullClusterRestart() throws Exception { PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class); int numberOfTasks = randomIntBetween(1, 10); long[] taskIds = new long[numberOfTasks]; - List futures = new ArrayList<>(numberOfTasks); + List>> futures = new ArrayList<>(numberOfTasks); for (int i = 0; i < numberOfTasks; i++) { - PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + PlainActionFuture> future = new PlainActionFuture<>(); futures.add(future); - service.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + service.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); } for (int i = 0; i < numberOfTasks; i++) { - taskIds[i] = futures.get(i).get(); + taskIds[i] = futures.get(i).get().getId(); } PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 7c6f7411624f4..399fd3053030b 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.PersistentTasksService.WaitForPersistentTaskStatusListener; import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; @@ -66,22 +67,20 @@ public void cleanup() throws Exception { assertNoRunningTasks(); } - public static class PersistentTaskOperationFuture extends PlainActionFuture implements WaitForPersistentTaskStatusListener { - @Override - public void onResponse(long taskId) { - set(taskId); - } + public static class WaitForPersistentTaskStatusFuture + extends PlainActionFuture> + implements WaitForPersistentTaskStatusListener { } public void testPersistentActionFailure() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); - persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - long taskId = future.get(); + PlainActionFuture> future = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get().getId(); assertBusy(() -> { // Wait for the task to start assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() - .getTasks().size(), equalTo(1)); + .getTasks().size(), equalTo(1)); }); TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); @@ -105,13 +104,13 @@ public void testPersistentActionFailure() throws Exception { public void testPersistentActionCompletion() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); - persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - long taskId = future.get(); + PlainActionFuture> future = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get().getId(); assertBusy(() -> { // Wait for the task to start assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() - .getTasks().size(), equalTo(1)); + .getTasks().size(), equalTo(1)); }); TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); @@ -124,11 +123,11 @@ public void testPersistentActionCompletion() throws Exception { public void testPersistentActionWithNoAvailableNode() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); + PlainActionFuture> future = new PlainActionFuture<>(); TestRequest testRequest = new TestRequest("Blah"); testRequest.setExecutorNodeAttr("test"); - persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, testRequest, future); - long taskId = future.get(); + persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, testRequest, future); + long taskId = future.get().getId(); Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); String newNode = internalCluster().startNode(nodeSettings); @@ -136,7 +135,7 @@ public void testPersistentActionWithNoAvailableNode() throws Exception { assertBusy(() -> { // Wait for the task to start assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() - .size(), equalTo(1)); + .size(), equalTo(1)); }); TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); @@ -153,21 +152,21 @@ public void testPersistentActionWithNoAvailableNode() throws Exception { }); // Remove the persistent task - PersistentTaskOperationFuture removeFuture = new PersistentTaskOperationFuture(); - persistentTasksService.removeTask(taskId, removeFuture); - assertEquals(removeFuture.get(), (Long) taskId); + PlainActionFuture> removeFuture = new PlainActionFuture<>(); + persistentTasksService.cancelPersistentTask(taskId, removeFuture); + assertEquals(removeFuture.get().getId(), taskId); } public void testPersistentActionStatusUpdate() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PersistentTaskOperationFuture future = new PersistentTaskOperationFuture(); - persistentTasksService.createPersistentActionTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - long taskId = future.get(); + PlainActionFuture> future = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get().getId(); assertBusy(() -> { // Wait for the task to start assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() - .size(), equalTo(1)); + .size(), equalTo(1)); }); TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") .get().getTasks().get(0); @@ -185,27 +184,27 @@ public void testPersistentActionStatusUpdate() throws Exception { .get().getTasks().size(), equalTo(1)); int finalI = i; - PersistentTaskOperationFuture future1 = new PersistentTaskOperationFuture(); + WaitForPersistentTaskStatusFuture future1 = new WaitForPersistentTaskStatusFuture<>(); persistentTasksService.waitForPersistentTaskStatus(taskId, - task -> task != null && task.isCurrentStatus()&& task.getStatus().toString() != null && + task -> task != null && task.isCurrentStatus() && task.getStatus().toString() != null && task.getStatus().toString().equals("{\"phase\":\"phase " + (finalI + 1) + "\"}"), TimeValue.timeValueSeconds(10), future1); - assertThat(future1.get(), equalTo(taskId)); + assertThat(future1.get().getId(), equalTo(taskId)); } - PersistentTaskOperationFuture future1 = new PersistentTaskOperationFuture(); + WaitForPersistentTaskStatusFuture future1 = new WaitForPersistentTaskStatusFuture<>(); persistentTasksService.waitForPersistentTaskStatus(taskId, task -> false, TimeValue.timeValueMillis(10), future1); assertThrows(future1, IllegalStateException.class, "timed out after 10ms"); - PersistentTaskOperationFuture failedUpdateFuture = new PersistentTaskOperationFuture(); + PlainActionFuture> failedUpdateFuture = new PlainActionFuture<>(); persistentTasksService.updateStatus(taskId, -1, new Status("should fail"), failedUpdateFuture); assertThrows(failedUpdateFuture, ResourceNotFoundException.class, "the task with id " + taskId + " and allocation id -1 doesn't exist"); // Wait for the task to disappear - PersistentTaskOperationFuture future2 = new PersistentTaskOperationFuture(); + WaitForPersistentTaskStatusFuture future2 = new WaitForPersistentTaskStatusFuture<>(); persistentTasksService.waitForPersistentTaskStatus(taskId, Objects::isNull, TimeValue.timeValueSeconds(10), future2); logger.info("Completing the running task"); @@ -213,7 +212,7 @@ public void testPersistentActionStatusUpdate() throws Exception { assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) .get().getTasks().size(), equalTo(1)); - assertThat(future2.get(), equalTo(taskId)); + assertThat(future2.get(), nullValue()); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java index 691077c80f2f4..8d1e4796ac1b8 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java @@ -18,17 +18,37 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.util.Collections; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; public class PersistentTasksExecutorResponseTests extends AbstractStreamableTestCase { @Override protected PersistentTaskResponse createTestInstance() { - return new PersistentTaskResponse(randomLong()); + if (randomBoolean()) { + return new PersistentTaskResponse( + new PersistentTask(randomLong(), randomAsciiOfLength(10), + new TestPersistentTasksPlugin.TestRequest("test"), + PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT)); + } else { + return new PersistentTaskResponse(null); + } } @Override protected PersistentTaskResponse createBlankInstance() { return new PersistentTaskResponse(); } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.singletonList( + new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, TestPersistentTasksPlugin.TestPersistentTasksExecutor.NAME, TestPersistentTasksPlugin.TestRequest::new) + )); + } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 576b7c6172216..0a147e83e63d4 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; @@ -35,8 +36,8 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; -import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; import java.io.IOException; @@ -163,10 +164,10 @@ public void testStartTask() throws Exception { public void testTaskCancellation() { ClusterService clusterService = createClusterService(); AtomicLong capturedTaskId = new AtomicLong(); - AtomicReference capturedListener = new AtomicReference<>(); + AtomicReference> capturedListener = new AtomicReference<>(); PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null, null) { @Override - public void sendCancellation(long taskId, PersistentTaskOperationListener listener) { + public void sendCancellation(long taskId, ActionListener listener) { capturedTaskId.set(taskId); capturedListener.set(listener); } @@ -217,7 +218,7 @@ public void sendCancellation(long taskId, PersistentTaskOperationListener listen // That should trigger cancellation request assertThat(capturedTaskId.get(), equalTo(localId)); // Notify successful cancellation - capturedListener.get().onResponse(localId); + capturedListener.get().onResponse(new CancelTasksResponse()); // finish or fail task if (randomBoolean()) { @@ -240,11 +241,12 @@ public void testNotificationFailure() throws Exception { ClusterService clusterService = createClusterService(); AtomicLong capturedTaskId = new AtomicLong(-1L); AtomicReference capturedException = new AtomicReference<>(); - AtomicReference capturedListener = new AtomicReference<>(); + AtomicReference>> capturedListener = new AtomicReference<>(); PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, null, null) { @Override - public void sendCompletionNotification(long taskId, Exception failure, PersistentTaskOperationListener listener) { + public void sendCompletionNotification(long taskId, Exception failure, + ActionListener> listener) { capturedTaskId.set(taskId); capturedException.set(failure); capturedListener.set(listener); @@ -297,7 +299,8 @@ public void sendCompletionNotification(long taskId, Exception failure, Persisten long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); // This time acknowledge notification - capturedListener.get().onResponse(id); + capturedListener.get().onResponse( + new PersistentTask<>(1, TestPersistentTasksPlugin.TestPersistentTasksExecutor.NAME, new TestRequest(), null)); // Reallocate failed task to another node state = newClusterState; diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 62288ed38e35d..4791f0c91d70f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -63,7 +63,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; -import org.elasticsearch.persistent.PersistentTasksService.PersistentTaskOperationListener; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.security.InternalClient; import java.io.IOException; @@ -94,7 +94,6 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { return Arrays.asList( new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class), - new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), new ActionHandler<>(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class), new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), new ActionHandler<>(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class) @@ -107,8 +106,7 @@ public Collection createComponents(Client client, ClusterService cluster NamedXContentRegistry xContentRegistry) { InternalClient internalClient = new InternalClient(Settings.EMPTY, threadPool, client); PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, internalClient); - TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, persistentTasksService, - clusterService); + TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, clusterService); PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(testPersistentAction)); return Arrays.asList( @@ -320,9 +318,8 @@ public static class TestPersistentTasksExecutor extends PersistentTasksExecutor< public static final String NAME = "cluster:admin/persistent/test"; private final ClusterService clusterService; - public TestPersistentTasksExecutor(Settings settings, PersistentTasksService persistentTasksService, - ClusterService clusterService) { - super(settings, NAME, persistentTasksService, ThreadPool.Names.GENERIC); + public TestPersistentTasksExecutor(Settings settings, ClusterService clusterService) { + super(settings, NAME, ThreadPool.Names.GENERIC); this.clusterService = clusterService; } @@ -368,9 +365,9 @@ protected void nodeOperation(AllocatedPersistentTask task, TestRequest request, CountDownLatch latch = new CountDownLatch(1); Status status = new Status("phase " + phase.incrementAndGet()); logger.info("updating the task status to {}", status); - task.updatePersistentStatus(status, new PersistentTaskOperationListener() { + task.updatePersistentStatus(status, new ActionListener>() { @Override - public void onResponse(long taskId) { + public void onResponse(PersistentTask persistentTask) { logger.info("updating was successful"); latch.countDown(); } From fab0dc449a2f57778a92bb6c847925a2d9b60efa Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 4 Apr 2017 21:31:17 +0200 Subject: [PATCH 23/50] Remove PersistentTask#isCurrentStatus() usages --- .../elasticsearch/persistent/AllocatedPersistentTask.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index b2115e5dd95ea..933586956d1a2 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -19,7 +19,6 @@ package org.elasticsearch.persistent; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -116,6 +115,10 @@ public State getState() { return state.get(); } + public long getAllocationId() { + return allocationId; + } + public enum State { STARTED, // the task is currently running CANCELLED, // the task is cancelled From 95c6005f6fbb06e64843e0115fd83c6a15c5ed5c Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 10 Apr 2017 12:26:09 -0400 Subject: [PATCH 24/50] Persistent Tasks: remove retries on notification failures (#977) Retries should be already handled by TransportMasterNodeAction, there is no need to introduce another retry layer in Persistent Tasks code. --- .../persistent/AllocatedPersistentTask.java | 24 +-- .../CreatePersistentTaskAction.java | 2 +- .../PersistentTasksClusterService.java | 1 - .../PersistentTasksNodeService.java | 144 +++++++----------- .../persistent/PersistentTasksService.java | 4 +- .../PersistentTasksNodeServiceTests.java | 94 +----------- 6 files changed, 75 insertions(+), 194 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index 933586956d1a2..f2e043000e00e 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -87,24 +87,12 @@ public Exception getFailure() { return failure; } - boolean startNotification(Exception failure) { - boolean result = state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.FAILED); - if (result) { + State markAsCompleted(Exception failure) { + State prevState = state.getAndSet(AllocatedPersistentTask.State.COMPLETED); + if (prevState == State.STARTED || prevState == State.CANCELLED) { this.failure = failure; } - return result; - } - - boolean notificationFailed() { - return state.compareAndSet(AllocatedPersistentTask.State.FAILED, AllocatedPersistentTask.State.FAILED_NOTIFICATION); - } - - boolean restartCompletionNotification() { - return state.compareAndSet(AllocatedPersistentTask.State.FAILED_NOTIFICATION, AllocatedPersistentTask.State.FAILED); - } - - boolean markAsNotified() { - return state.compareAndSet(AllocatedPersistentTask.State.FAILED, AllocatedPersistentTask.State.NOTIFIED); + return prevState; } boolean markAsCancelled() { @@ -122,8 +110,6 @@ public long getAllocationId() { public enum State { STARTED, // the task is currently running CANCELLED, // the task is cancelled - FAILED, // the task is done running and trying to notify caller - FAILED_NOTIFICATION, // the caller notification failed - NOTIFIED // the caller was notified, the task can be removed + COMPLETED // the task is done running and trying to notify caller } } diff --git a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java index 892c51d41b86a..8ec680e702d92 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java @@ -172,7 +172,7 @@ public TransportAction(Settings settings, TransportService transportService, Clu this.persistentTasksClusterService = persistentTasksClusterService; NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool); clusterService.addListener(new PersistentTasksNodeService(settings, persistentTasksService, persistentTasksExecutorRegistry, - transportService.getTaskManager(), threadPool, executor)); + transportService.getTaskManager(), executor)); } @Override diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 6e391dc4a66e1..96c1a5afc94cf 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; -import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index 5fdbaf721fd76..c3fb60bf74878 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -30,12 +30,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; @@ -57,20 +55,17 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu private final PersistentTasksService persistentTasksService; private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry; private final TaskManager taskManager; - private final ThreadPool threadPool; private final NodePersistentTasksExecutor nodePersistentTasksExecutor; public PersistentTasksNodeService(Settings settings, PersistentTasksService persistentTasksService, PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, - TaskManager taskManager, ThreadPool threadPool, - NodePersistentTasksExecutor nodePersistentTasksExecutor) { + TaskManager taskManager, NodePersistentTasksExecutor nodePersistentTasksExecutor) { super(settings); this.persistentTasksService = persistentTasksService; this.persistentTasksExecutorRegistry = persistentTasksExecutorRegistry; this.taskManager = taskManager; - this.threadPool = threadPool; this.nodePersistentTasksExecutor = nodePersistentTasksExecutor; } @@ -79,6 +74,29 @@ public void clusterChanged(ClusterChangedEvent event) { PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); PersistentTasksCustomMetaData previousTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + // Cluster State Local State Local Action + // STARTED NULL Create as STARTED, Start + // STARTED STARTED Noop - running + // STARTED COMPLETED Noop - waiting for notification ack + + // NULL NULL Noop - nothing to do + // NULL STARTED Remove locally, Mark as CANCELLED, Cancel + // NULL COMPLETED Remove locally + + // Master states: + // NULL - doesn't exist in the cluster state + // STARTED - exist in the cluster state + + // Local state: + // NULL - we don't have task registered locally in runningTasks + // STARTED - registered in TaskManager, requires master notification when finishes + // CANCELLED - registered in TaskManager, doesn't require master notification when finishes + // COMPLETED - not registered in TaskManager, notified, waiting for master to remove it from CS so we can remove locally + + // When task finishes if it is marked as STARTED or CANCELLED it is marked as COMPLETED and unregistered, + // If the task was STARTED, the master notification is also triggered (this is handled by unregisterTask() method, which is + // triggered by PersistentTaskListener + if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { // We have some changes let's check if they are related to our node String localNodeId = event.state().getNodes().getLocalNodeId(); @@ -94,11 +112,6 @@ public void clusterChanged(ClusterChangedEvent event) { } else { // The task is still running notVisitedTasks.remove(persistentTaskId); - if (persistentTask.getState() == AllocatedPersistentTask.State.FAILED_NOTIFICATION) { - // We tried to notify the master about this task before but the notification failed and - // the master doesn't seem to know about it - retry notification - restartCompletionNotification(persistentTask); - } } } } @@ -106,17 +119,13 @@ public void clusterChanged(ClusterChangedEvent event) { for (PersistentTaskId id : notVisitedTasks) { AllocatedPersistentTask task = runningTasks.get(id); - if (task.getState() == AllocatedPersistentTask.State.NOTIFIED || task.getState() == AllocatedPersistentTask.State.FAILED) { + if (task.getState() == AllocatedPersistentTask.State.COMPLETED) { // Result was sent to the caller and the caller acknowledged acceptance of the result finishTask(id); - } else if (task.getState() == AllocatedPersistentTask.State.FAILED_NOTIFICATION) { - // We tried to send result to master, but it failed and master doesn't know about this task - // this shouldn't really happen, unless this node is severally out of sync with the master - logger.warn("failed to notify master about task {}", task.getPersistentTaskId()); - finishTask(id); } else { // task is running locally, but master doesn't know about it - that means that the persistent task was removed // cancel the task without notifying master + logger.trace("Found unregistered persistent task with id {} - cancelling ", id); cancelTask(id); } } @@ -149,6 +158,9 @@ private void startTask(PersistentTask() { + // Cancel the local task using the task manager + persistentTasksService.sendTaskManagerCancellation(task.getId(), new ActionListener() { @Override public void onResponse(CancelTasksResponse cancelTasksResponse) { logger.trace("Persistent task with id {} was cancelled", task.getId()); @@ -177,47 +194,30 @@ public void onFailure(Exception e) { } } - - private void restartCompletionNotification(AllocatedPersistentTask task) { - logger.trace("resending notification for task {}", task.getPersistentTaskId()); - if (task.getState() == AllocatedPersistentTask.State.CANCELLED) { + private void unregisterTask(AllocatedPersistentTask task, Exception e) { + AllocatedPersistentTask.State prevState = task.markAsCompleted(e); + if (prevState == AllocatedPersistentTask.State.CANCELLED) { + // The task was cancelled by master - no need to send notifications taskManager.unregister(task); - } else { - if (task.restartCompletionNotification()) { - // Need to fork otherwise: java.lang.AssertionError: should not be called by a cluster state applier. - // reason [the applied cluster state is not yet available]) - PublishedResponseListener listener = new PublishedResponseListener(task); - try { - threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - - @Override - protected void doRun() throws Exception { - persistentTasksService.sendCompletionNotification(task.getPersistentTaskId(), task.getFailure(), listener); - } - }); - } catch (Exception e) { - listener.onFailure(e); + } else if (prevState == AllocatedPersistentTask.State.STARTED) { + // The task finished locally, but master doesn't know about it - we need notify the master before we can unregister it + logger.trace("sending notification for completed task {}", task.getPersistentTaskId()); + persistentTasksService.sendCompletionNotification(task.getPersistentTaskId(), e, new ActionListener>() { + @Override + public void onResponse(PersistentTask persistentTask) { + logger.trace("notification for task {} was successful", task.getId()); + taskManager.unregister(task); } - } else { - logger.warn("attempt to resend notification for task {} in the {} state", task.getPersistentTaskId(), task.getState()); - } - } - } - private void startCompletionNotification(AllocatedPersistentTask task, Exception e) { - if (task.getState() == AllocatedPersistentTask.State.CANCELLED) { - taskManager.unregister(task); + @Override + public void onFailure(Exception e) { + logger.warn((Supplier) () -> + new ParameterizedMessage("notification for task {} failed", task.getPersistentTaskId()), e); + taskManager.unregister(task); + } + }); } else { - logger.trace("sending notification for failed task {}", task.getPersistentTaskId()); - if (task.startNotification(e)) { - persistentTasksService.sendCompletionNotification(task.getPersistentTaskId(), e, new PublishedResponseListener(task)); - } else { - logger.warn("attempt to send notification for task {} in the {} state", task.getPersistentTaskId(), task.getState()); - } + logger.warn("attempt to complete task {} in the {} state", task.getPersistentTaskId(), prevState); } } @@ -230,7 +230,7 @@ private class PersistentTaskListener implements ActionListener { @Override public void onResponse(Empty response) { - startCompletionNotification(task, null); + unregisterTask(task, null); } @Override @@ -243,38 +243,12 @@ public void onFailure(Exception e) { task.getPersistentTaskId(), task.getReasonCancelled()), e); } if (CancelTasksRequest.DEFAULT_REASON.equals(task.getReasonCancelled())) { - startCompletionNotification(task, null); + unregisterTask(task, null); } else { - startCompletionNotification(task, e); + unregisterTask(task, e); } } else { - startCompletionNotification(task, e); - } - } - } - - private class PublishedResponseListener implements ActionListener> { - private final AllocatedPersistentTask task; - - PublishedResponseListener(final AllocatedPersistentTask task) { - this.task = task; - } - - - @Override - public void onResponse(PersistentTask persistentTask) { - logger.trace("notification for task {} was successful", task.getId()); - if (task.markAsNotified() == false) { - logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getPersistentTaskId(), task.getState()); - } - taskManager.unregister(task); - } - - @Override - public void onFailure(Exception e) { - logger.warn((Supplier) () -> new ParameterizedMessage("notification for task {} failed - retrying", task.getPersistentTaskId()), e); - if (task.notificationFailed() == false) { - logger.warn("attempt to mark restart notification for task {} in the {} state failed", task.getPersistentTaskId(), task.getState()); + unregisterTask(task, e); } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 064b89794394c..725cbc2672fed 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -83,9 +83,9 @@ public void sendCompletionNotification(long taskId, Exception failure, ActionLis } /** - * Cancels the persistent task. + * Cancels a locally running task using the task manager */ - void sendCancellation(long taskId, ActionListener listener) { + void sendTaskManagerCancellation(long taskId, ActionListener listener) { DiscoveryNode localNode = clusterService.localNode(); CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 0a147e83e63d4..42f5dff2e5e65 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -82,7 +82,7 @@ public void testStartTask() throws Exception { int nonLocalNodesCount = randomInt(10); MockExecutor executor = new MockExecutor(); PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, - registry, new TaskManager(Settings.EMPTY), mock(ThreadPool.class), executor); + registry, new TaskManager(Settings.EMPTY), executor); ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); @@ -167,10 +167,15 @@ public void testTaskCancellation() { AtomicReference> capturedListener = new AtomicReference<>(); PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null, null) { @Override - public void sendCancellation(long taskId, ActionListener listener) { + public void sendTaskManagerCancellation(long taskId, ActionListener listener) { capturedTaskId.set(taskId); capturedListener.set(listener); } + + @Override + public void sendCompletionNotification(long taskId, Exception failure, ActionListener> listener) { + fail("Shouldn't be called during Cluster State cancellation"); + } }; @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); @@ -181,7 +186,7 @@ public void sendCancellation(long taskId, ActionListener li MockExecutor executor = new MockExecutor(); TaskManager taskManager = new TaskManager(Settings.EMPTY); PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, - registry, taskManager, mock(ThreadPool.class), executor); + registry, taskManager, executor); ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) .build(); @@ -232,89 +237,6 @@ public void sendCancellation(long taskId, ActionListener li } - public void testNotificationFailure() throws Exception { - Settings settings = Settings.builder() - .put("node.name", PersistentTasksNodeServiceTests.class.getSimpleName()) - .build(); - ThreadPool threadPool = new ThreadPool(settings); - try { - ClusterService clusterService = createClusterService(); - AtomicLong capturedTaskId = new AtomicLong(-1L); - AtomicReference capturedException = new AtomicReference<>(); - AtomicReference>> capturedListener = new AtomicReference<>(); - PersistentTasksService persistentTasksService = - new PersistentTasksService(Settings.EMPTY, clusterService, null, null) { - @Override - public void sendCompletionNotification(long taskId, Exception failure, - ActionListener> listener) { - capturedTaskId.set(taskId); - capturedException.set(failure); - capturedListener.set(listener); - } - }; - @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); - when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - when(action.getTaskName()).thenReturn("test"); - PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, - Collections.singletonList(action)); - - int nonLocalNodesCount = randomInt(10); - MockExecutor executor = new MockExecutor(); - TaskManager taskManager = new TaskManager(Settings.EMPTY); - PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, - registry, taskManager, threadPool, executor); - - ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) - .build(); - - ClusterState newClusterState = state; - // Allocate first task - state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "this_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - - // Fail the task - executor.get(0).listener.onFailure(new RuntimeException("test failure")); - - // Check that notification was sent - assertThat(capturedException.get().getMessage(), equalTo("test failure")); - capturedException.set(null); - - // Simulate failure to notify - capturedListener.get().onFailure(new IOException("simulated notification failure")); - - // Allocate another task - state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "other_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - - // Check that notification was sent again - assertBusy(() -> { - assertNotNull(capturedException.get()); - assertThat(capturedException.get().getMessage(), equalTo("test failure")); - }); - - // Check the the task is still known by the task manager - assertThat(taskManager.getTasks().size(), equalTo(1)); - long id = taskManager.getTasks().values().iterator().next().getParentTaskId().getId(); - - // This time acknowledge notification - capturedListener.get().onResponse( - new PersistentTask<>(1, TestPersistentTasksPlugin.TestPersistentTasksExecutor.NAME, new TestRequest(), null)); - - // Reallocate failed task to another node - state = newClusterState; - newClusterState = reallocateTask(state, id, "other_node"); - coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); - - // Check the the task is now removed from task manager - assertThat(taskManager.getTasks().values(), empty()); - } finally { - assertTrue(ESTestCase.terminate(threadPool)); - } - - } - private ClusterState addTask(ClusterState state, String action, Request request, String node) { PersistentTasksCustomMetaData.Builder builder = From 0a1abd430dd911686cca996fe087931a2e01b65f Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 10 Apr 2017 17:32:30 -0400 Subject: [PATCH 25/50] Persistent Tasks: remove listener from PersistentTasksExecutor#nodeOperation (#1032) Instead of having a separate listener for indicating that the current task is finished, this commit is switching to use allocated object itself. --- .../persistent/AllocatedPersistentTask.java | 73 +++++++++++++--- .../NodePersistentTasksExecutor.java | 11 +-- .../persistent/PersistentTasksExecutor.java | 7 +- .../PersistentTasksNodeService.java | 87 ++----------------- .../PersistentTasksNodeServiceTests.java | 21 ++--- .../persistent/TestPersistentTasksPlugin.java | 15 ++-- 6 files changed, 90 insertions(+), 124 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index f2e043000e00e..733ac9774cb7a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -18,11 +18,17 @@ */ package org.elasticsearch.persistent; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.common.Nullable; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; import java.util.concurrent.atomic.AtomicReference; @@ -38,6 +44,8 @@ public class AllocatedPersistentTask extends CancellableTask { private Exception failure; private PersistentTasksService persistentTasksService; + private Logger logger; + private TaskManager taskManager; public AllocatedPersistentTask(long id, String type, String action, String description, TaskId parentTask) { @@ -66,7 +74,7 @@ public Status getStatus() { /** * Updates the persistent state for the corresponding persistent task. - * + *

* This doesn't affect the status of this allocated task. */ public void updatePersistentStatus(Task.Status status, ActionListener> listener) { @@ -77,8 +85,11 @@ public long getPersistentTaskId() { return persistentTaskId; } - void init(PersistentTasksService persistentTasksService, long persistentTaskId, long allocationId) { + void init(PersistentTasksService persistentTasksService, TaskManager taskManager, Logger logger, long persistentTaskId, long + allocationId) { this.persistentTasksService = persistentTasksService; + this.logger = logger; + this.taskManager = taskManager; this.persistentTaskId = persistentTaskId; this.allocationId = allocationId; } @@ -87,16 +98,8 @@ public Exception getFailure() { return failure; } - State markAsCompleted(Exception failure) { - State prevState = state.getAndSet(AllocatedPersistentTask.State.COMPLETED); - if (prevState == State.STARTED || prevState == State.CANCELLED) { - this.failure = failure; - } - return prevState; - } - boolean markAsCancelled() { - return state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.CANCELLED); + return state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.PENDING_CANCEL); } public State getState() { @@ -109,7 +112,53 @@ public long getAllocationId() { public enum State { STARTED, // the task is currently running - CANCELLED, // the task is cancelled + PENDING_CANCEL, // the task is cancelled on master, cancelling it locally COMPLETED // the task is done running and trying to notify caller } + + public void markAsCompleted() { + completeAndNotifyIfNeeded(null); + } + + public void markAsFailed(Exception e) { + if (CancelTasksRequest.DEFAULT_REASON.equals(getReasonCancelled())) { + completeAndNotifyIfNeeded(null); + } else { + completeAndNotifyIfNeeded(e); + } + + } + + private void completeAndNotifyIfNeeded(@Nullable Exception failure) { + State prevState = state.getAndSet(AllocatedPersistentTask.State.COMPLETED); + if (prevState == State.COMPLETED) { + logger.warn("attempt to complete task {} in the {} state", getPersistentTaskId(), prevState); + } else { + if (failure != null) { + logger.warn((Supplier) () -> new ParameterizedMessage( + "task {} failed with an exception", getPersistentTaskId()), failure); + } + try { + this.failure = failure; + if (prevState == State.STARTED) { + logger.trace("sending notification for completed task {}", getPersistentTaskId()); + persistentTasksService.sendCompletionNotification(getPersistentTaskId(), failure, new + ActionListener>() { + @Override + public void onResponse(PersistentTasksCustomMetaData.PersistentTask persistentTask) { + logger.trace("notification for task {} was successful", getId()); + } + + @Override + public void onFailure(Exception e) { + logger.warn((Supplier) () -> + new ParameterizedMessage("notification for task {} failed", getPersistentTaskId()), e); + } + }); + } + } finally { + taskManager.unregister(this); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index d96540b6c9dfa..4e4c0692a95eb 100644 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -18,10 +18,8 @@ */ package org.elasticsearch.persistent; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse.Empty; /** * This component is responsible for execution of persistent tasks. @@ -37,21 +35,20 @@ public NodePersistentTasksExecutor(ThreadPool threadPool) { public void executeTask(Request request, AllocatedPersistentTask task, - PersistentTasksExecutor action, - ActionListener listener) { + PersistentTasksExecutor action) { threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { - listener.onFailure(e); + task.markAsFailed(e); } @SuppressWarnings("unchecked") @Override protected void doRun() throws Exception { try { - action.nodeOperation(task, request, listener); + action.nodeOperation(task, request); } catch (Exception ex) { - listener.onFailure(ex); + task.markAsFailed(ex); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 656b83187589f..93be9d068415b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -99,11 +99,10 @@ public void validate(Request request, ClusterState clusterState) { /** * This operation will be executed on the executor node. *

- * If nodeOperation throws an exception or triggers listener.onFailure() method, the task will be restarted, - * possibly on a different node. If listener.onResponse() is called, the task is considered to be successfully - * completed and will be removed from the cluster state and not restarted. + * NOTE: The nodeOperation has to throws an exception, trigger task.markAsCompleted() or task.completeAndNotifyIfNeeded() methods to + * indicate that the persistent task has finished. */ - protected abstract void nodeOperation(AllocatedPersistentTask task, Request request, ActionListener listener); + protected abstract void nodeOperation(AllocatedPersistentTask task, Request request); public String getExecutor() { return executor; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index c3fb60bf74878..1344d9b703c58 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -21,7 +21,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; @@ -32,9 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; @@ -80,7 +77,7 @@ public void clusterChanged(ClusterChangedEvent event) { // STARTED COMPLETED Noop - waiting for notification ack // NULL NULL Noop - nothing to do - // NULL STARTED Remove locally, Mark as CANCELLED, Cancel + // NULL STARTED Remove locally, Mark as PENDING_CANCEL, Cancel // NULL COMPLETED Remove locally // Master states: @@ -90,10 +87,10 @@ public void clusterChanged(ClusterChangedEvent event) { // Local state: // NULL - we don't have task registered locally in runningTasks // STARTED - registered in TaskManager, requires master notification when finishes - // CANCELLED - registered in TaskManager, doesn't require master notification when finishes + // PENDING_CANCEL - registered in TaskManager, doesn't require master notification when finishes // COMPLETED - not registered in TaskManager, notified, waiting for master to remove it from CS so we can remove locally - // When task finishes if it is marked as STARTED or CANCELLED it is marked as COMPLETED and unregistered, + // When task finishes if it is marked as STARTED or PENDING_CANCEL it is marked as COMPLETED and unregistered, // If the task was STARTED, the master notification is also triggered (this is handled by unregisterTask() method, which is // triggered by PersistentTaskListener @@ -121,7 +118,7 @@ public void clusterChanged(ClusterChangedEvent event) { AllocatedPersistentTask task = runningTasks.get(id); if (task.getState() == AllocatedPersistentTask.State.COMPLETED) { // Result was sent to the caller and the caller acknowledged acceptance of the result - finishTask(id); + runningTasks.remove(id); } else { // task is running locally, but master doesn't know about it - that means that the persistent task was removed // cancel the task without notifying master @@ -140,14 +137,13 @@ private void startTask(PersistentTask void startTask(PersistentTask>() { - @Override - public void onResponse(PersistentTask persistentTask) { - logger.trace("notification for task {} was successful", task.getId()); - taskManager.unregister(task); - } - - @Override - public void onFailure(Exception e) { - logger.warn((Supplier) () -> - new ParameterizedMessage("notification for task {} failed", task.getPersistentTaskId()), e); - taskManager.unregister(task); - } - }); - } else { - logger.warn("attempt to complete task {} in the {} state", task.getPersistentTaskId(), prevState); - } - } - - private class PersistentTaskListener implements ActionListener { - private final AllocatedPersistentTask task; - - PersistentTaskListener(final AllocatedPersistentTask task) { - this.task = task; - } - - @Override - public void onResponse(Empty response) { - unregisterTask(task, null); - } - - @Override - public void onFailure(Exception e) { - if (task.isCancelled()) { - // The task was explicitly cancelled - no need to restart it, just log the exception if it's not TaskCancelledException - if (e instanceof TaskCancelledException == false) { - logger.warn((Supplier) () -> new ParameterizedMessage( - "cancelled task {} failed with an exception, cancellation reason [{}]", - task.getPersistentTaskId(), task.getReasonCancelled()), e); - } - if (CancelTasksRequest.DEFAULT_REASON.equals(task.getReasonCancelled())) { - unregisterTask(task, null); - } else { - unregisterTask(task, e); - } - } else { - unregisterTask(task, e); - } - } - } - private static class PersistentTaskId { private final long id; private final long allocationId; diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 42f5dff2e5e65..c35432deddd2d 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; @@ -131,8 +130,8 @@ public void testStartTask() throws Exception { assertThat(executor.size(), equalTo(2)); // Finish both tasks - executor.get(0).listener.onFailure(new RuntimeException()); - executor.get(1).listener.onResponse(Empty.INSTANCE); + executor.get(0).task.markAsFailed(new RuntimeException()); + executor.get(1).task.markAsCompleted(); long failedTaskId = executor.get(0).task.getParentTaskId().getId(); long finishedTaskId = executor.get(1).task.getParentTaskId().getId(); executor.clear(); @@ -217,7 +216,7 @@ public void sendCompletionNotification(long taskId, Exception failure, ActionLis // Make sure it returns correct status assertThat(taskManager.getTasks().size(), equalTo(1)); - assertThat(taskManager.getTasks().values().iterator().next().getStatus().toString(), equalTo("{\"state\":\"CANCELLED\"}")); + assertThat(taskManager.getTasks().values().iterator().next().getStatus().toString(), equalTo("{\"state\":\"PENDING_CANCEL\"}")); // That should trigger cancellation request @@ -227,9 +226,9 @@ public void sendCompletionNotification(long taskId, Exception failure, ActionLis // finish or fail task if (randomBoolean()) { - executor.get(0).listener.onResponse(Empty.INSTANCE); + executor.get(0).task.markAsCompleted(); } else { - executor.get(0).listener.onFailure(new IOException("test")); + executor.get(0).task.markAsFailed(new IOException("test")); } // Check the the task is now removed from task manager @@ -265,14 +264,11 @@ private class Execution { private final PersistentTaskRequest request; private final AllocatedPersistentTask task; private final PersistentTasksExecutor holder; - private final ActionListener listener; - Execution(PersistentTaskRequest request, AllocatedPersistentTask task, PersistentTasksExecutor holder, - ActionListener listener) { + Execution(PersistentTaskRequest request, AllocatedPersistentTask task, PersistentTasksExecutor holder) { this.request = request; this.task = task; this.holder = holder; - this.listener = listener; } } @@ -285,9 +281,8 @@ private class MockExecutor extends NodePersistentTasksExecutor { @Override public void executeTask(Request request, AllocatedPersistentTask task, - PersistentTasksExecutor action, - ActionListener listener) { - executions.add(new Execution(request, task, action, listener)); + PersistentTasksExecutor action) { + executions.add(new Execution(request, task, action)); } public Execution get(int i) { diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 4791f0c91d70f..ae7b447a26d93 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -59,7 +59,6 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; @@ -340,7 +339,7 @@ public Assignment getAssignment(TestRequest request, ClusterState clusterState) } @Override - protected void nodeOperation(AllocatedPersistentTask task, TestRequest request, ActionListener listener) { + protected void nodeOperation(AllocatedPersistentTask task, TestRequest request) { logger.info("started node operation for the task {}", task); try { TestTask testTask = (TestTask) task; @@ -355,10 +354,10 @@ protected void nodeOperation(AllocatedPersistentTask task, TestRequest request, return; } if ("finish".equals(testTask.getOperation())) { - listener.onResponse(Empty.INSTANCE); + task.markAsCompleted(); return; } else if ("fail".equals(testTask.getOperation())) { - listener.onFailure(new RuntimeException("Simulating failure")); + task.markAsFailed(new RuntimeException("Simulating failure")); return; } else if ("update_status".equals(testTask.getOperation())) { testTask.setOperation(null); @@ -384,12 +383,12 @@ public void onFailure(Exception e) { // Cancellation make cause different ways for the task to finish if (randomBoolean()) { if (randomBoolean()) { - listener.onFailure(new TaskCancelledException(testTask.getReasonCancelled())); + task.markAsFailed(new TaskCancelledException(testTask.getReasonCancelled())); } else { - listener.onResponse(Empty.INSTANCE); + task.markAsCompleted(); } } else { - listener.onFailure(new RuntimeException(testTask.getReasonCancelled())); + task.markAsFailed(new RuntimeException(testTask.getReasonCancelled())); } return; } else { @@ -397,7 +396,7 @@ public void onFailure(Exception e) { } } } catch (InterruptedException e) { - listener.onFailure(e); + task.markAsFailed(e); } } } From 0a1f25588bc2b9835b8f671872b5b3d06d1f172f Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 11 Apr 2017 13:39:22 +0200 Subject: [PATCH 26/50] Added PersistentTasksService#waitForPersistentTasksStatus(...) method to allow callers to wait when an executor node has updated its task status. --- .../persistent/PersistentTasksService.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 725cbc2672fed..eb0f9a1a3e8a6 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -156,6 +156,32 @@ public void onTimeout(TimeValue timeout) { } } + public void waitForPersistentTasksStatus(Predicate predicate, + @Nullable TimeValue timeout, ActionListener listener) { + ClusterStateObserver stateObserver = new ClusterStateObserver(clusterService, timeout, + logger, threadPool.getThreadContext()); + if (predicate.test(stateObserver.setAndGetObservedState().metaData().custom(PersistentTasksCustomMetaData.TYPE))) { + listener.onResponse(true); + } else { + stateObserver.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + listener.onResponse(true); + } + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + listener.onFailure(new IllegalStateException("timed out after " + timeout)); + } + }, clusterState -> predicate.test(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE))); + } + } + public interface WaitForPersistentTaskStatusListener extends ActionListener> { default void onTimeout(TimeValue timeout) { From 6bfea09dd62f2a9f953a9f318ffedbcb4fb96d63 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 11 Apr 2017 12:24:54 -0400 Subject: [PATCH 27/50] Persistent Tasks: switch from long task ids to string task ids (#1035) This commit switches from long persistent task ids to caller-supplied string persistent task ids. --- .../persistent/AllocatedPersistentTask.java | 6 +- .../CompletionPersistentTaskAction.java | 18 +- .../CreatePersistentTaskAction.java | 29 +++- .../PersistentTasksClusterService.java | 25 +-- .../PersistentTasksCustomMetaData.java | 162 +++++++++--------- .../PersistentTasksNodeService.java | 70 +++----- .../persistent/PersistentTasksService.java | 13 +- .../RemovePersistentTaskAction.java | 16 +- .../UpdatePersistentTaskStatusAction.java | 14 +- .../CancelPersistentTaskRequestTests.java | 6 +- .../PersistentTasksClusterServiceTests.java | 5 +- .../PersistentTasksCustomMetaDataTests.java | 79 ++++----- .../PersistentTasksExecutorFullRestartIT.java | 8 +- .../persistent/PersistentTasksExecutorIT.java | 52 +++++- .../PersistentTasksExecutorResponseTests.java | 5 +- .../PersistentTasksNodeServiceTests.java | 20 +-- .../RestartPersistentTaskRequestTests.java | 4 +- .../StartPersistentActionRequestTests.java | 3 +- .../UpdatePersistentTaskRequestTests.java | 3 +- 19 files changed, 287 insertions(+), 251 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index 733ac9774cb7a..dee95ac9814ba 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -36,7 +36,7 @@ * Represents a executor node operation that corresponds to a persistent task */ public class AllocatedPersistentTask extends CancellableTask { - private long persistentTaskId; + private String persistentTaskId; private long allocationId; private final AtomicReference state; @@ -81,11 +81,11 @@ public void updatePersistentStatus(Task.Status status, ActionListener { - private long taskId; + private String taskId; private Exception exception; @@ -77,7 +79,7 @@ public Request() { } - public Request(long taskId, Exception exception) { + public Request(String taskId, Exception exception) { this.taskId = taskId; this.exception = exception; } @@ -85,20 +87,24 @@ public Request(long taskId, Exception exception) { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - taskId = in.readLong(); + taskId = in.readString(); exception = in.readException(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeLong(taskId); + out.writeString(taskId); out.writeException(exception); } @Override public ActionRequestValidationException validate() { - return null; + ActionRequestValidationException validationException = null; + if (taskId == null) { + validationException = addValidationError("task id is missing", validationException); + } + return validationException; } @Override @@ -106,7 +112,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return taskId == request.taskId && + return Objects.equals(taskId, request.taskId) && Objects.equals(exception, request.exception); } diff --git a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java index 8ec680e702d92..6c2b8311d6192 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java @@ -66,6 +66,8 @@ public PersistentTaskResponse newResponse() { public static class Request extends MasterNodeRequest { + private String taskId; + private String action; private PersistentTaskRequest request; @@ -74,7 +76,8 @@ public Request() { } - public Request(String action, PersistentTaskRequest request) { + public Request(String taskId, String action, PersistentTaskRequest request) { + this.taskId = taskId; this.action = action; this.request = request; } @@ -82,6 +85,7 @@ public Request(String action, PersistentTaskRequest request) { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); + taskId = in.readString(); action = in.readString(); request = in.readNamedWriteable(PersistentTaskRequest.class); } @@ -89,6 +93,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + out.writeString(taskId); out.writeString(action); out.writeNamedWriteable(request); } @@ -96,6 +101,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; + if (this.taskId == null) { + validationException = addValidationError("task id must be specified", validationException); + } if (this.action == null) { validationException = addValidationError("action must be specified", validationException); } @@ -110,13 +118,13 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request1 = (Request) o; - return Objects.equals(action, request1.action) && + return Objects.equals(taskId, request1.taskId) && Objects.equals(action, request1.action) && Objects.equals(request, request1.request); } @Override public int hashCode() { - return Objects.hash(action, request); + return Objects.hash(taskId, action, request); } public String getAction() { @@ -127,6 +135,14 @@ public void setAction(String action) { this.action = action; } + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + public PersistentTaskRequest getRequest() { return request; } @@ -144,6 +160,11 @@ protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction super(client, action, new Request()); } + public RequestBuilder setTaskId(String taskId) { + request.setTaskId(taskId); + return this; + } + public RequestBuilder setAction(String action) { request.setAction(action); return this; @@ -194,7 +215,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.createPersistentTask(request.action, request.request, + persistentTasksClusterService.createPersistentTask(request.taskId, request.action, request.request, new ActionListener>() { @Override diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 96c1a5afc94cf..fb14922ae3eb7 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -20,6 +20,7 @@ package org.elasticsearch.persistent; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -60,15 +61,19 @@ public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorR * @param request request * @param listener the listener that will be called when task is started */ - public void createPersistentTask(String action, Request request, + public void createPersistentTask(String taskId, String action, Request request, ActionListener> listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksCustomMetaData.Builder builder = builder(currentState); + if (builder.hasTask(taskId)) { + throw new ResourceAlreadyExistsException("task with id {" + taskId + "} already exist"); + } validate(action, clusterService.state(), request); final Assignment assignment; assignment = getAssignement(action, currentState, request); - return update(currentState, builder(currentState).addTask(action, request, assignment)); + return update(currentState, builder.addTask(taskId, action, request, assignment)); } @Override @@ -81,7 +86,7 @@ public void onFailure(String source, Exception e) { public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks != null) { - listener.onResponse(tasks.getTask(tasks.getCurrentId())); + listener.onResponse(tasks.getTask(taskId)); } else { listener.onResponse(null); } @@ -97,7 +102,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @param failure the reason for restarting the task or null if the task completed successfully * @param listener the listener that will be called when task is removed */ - public void completePersistentTask(long id, Exception failure, ActionListener> listener) { + public void completePersistentTask(String id, Exception failure, ActionListener> listener) { final String source; if (failure != null) { logger.warn("persistent task " + id + " failed", failure); @@ -138,7 +143,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS * @param id the id of a persistent task * @param listener the listener that will be called when task is removed */ - public void removePersistentTask(long id, ActionListener> listener) { + public void removePersistentTask(String id, ActionListener> listener) { clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -166,12 +171,12 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS /** * Update task status * - * @param id the id of a persistent task - * @param allocationId the expected allocation id of the persistent task - * @param status new status - * @param listener the listener that will be called when task is removed + * @param id the id of a persistent task + * @param allocationId the expected allocation id of the persistent task + * @param status new status + * @param listener the listener that will be called when task is removed */ - public void updatePersistentTaskStatus(long id, long allocationId, Task.Status status, ActionListener> listener) { + public void updatePersistentTaskStatus(String id, long allocationId, Task.Status status, ActionListener> listener) { clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 5366b432b95aa..70dce599cbcec 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.ClusterState; @@ -47,7 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiFunction; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -63,12 +65,12 @@ public final class PersistentTasksCustomMetaData extends AbstractNamedDiffable> tasks; + private final Map> tasks; - private final long currentId; + private final long lastAllocationId; - public PersistentTasksCustomMetaData(long currentId, Map> tasks) { - this.currentId = currentId; + public PersistentTasksCustomMetaData(long lastAllocationId, Map> tasks) { + this.lastAllocationId = lastAllocationId; this.tasks = tasks; } @@ -87,7 +89,7 @@ public PersistentTasksCustomMetaData(long currentId, Map static { // Tasks parser initialization - PERSISTENT_TASKS_PARSER.declareLong(Builder::setCurrentId, new ParseField("current_id")); + PERSISTENT_TASKS_PARSER.declareLong(Builder::setLastAllocationId, new ParseField("last_allocation_id")); PERSISTENT_TASKS_PARSER.declareObjectArray(Builder::setTasks, PERSISTENT_TASK_PARSER, new ParseField("tasks")); // Assignment parser @@ -95,7 +97,7 @@ public PersistentTasksCustomMetaData(long currentId, Map ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("explanation")); // Task parser initialization - PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setId, new ParseField("id")); + PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setId, new ParseField("id")); PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setTaskName, new ParseField("name")); PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id")); PERSISTENT_TASK_PARSER.declareNamedObjects( @@ -124,11 +126,11 @@ public Collection> tasks() { return this.tasks.values(); } - public Map> taskMap() { + public Map> taskMap() { return this.tasks; } - public PersistentTask getTask(long id) { + public PersistentTask getTask(String id) { return this.tasks.get(id); } @@ -150,13 +152,13 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; PersistentTasksCustomMetaData that = (PersistentTasksCustomMetaData) o; - return currentId == that.currentId && + return lastAllocationId == that.lastAllocationId && Objects.equals(tasks, that.tasks); } @Override public int hashCode() { - return Objects.hash(tasks, currentId); + return Objects.hash(tasks, lastAllocationId); } @Override @@ -184,10 +186,10 @@ public static PersistentTasksCustomMetaData fromXContent(XContentParser parser) } @SuppressWarnings("unchecked") - public static PersistentTask getTaskWithId(ClusterState clusterState, long taskId) { + public static PersistentTask getTaskWithId(ClusterState clusterState, String taskId) { PersistentTasksCustomMetaData tasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks != null) { - return (PersistentTask)tasks.getTask(taskId); + return (PersistentTask) tasks.getTask(taskId); } return null; } @@ -232,7 +234,7 @@ public boolean isAssigned() { @Override public String toString() { - return "node: [" + executorNode + "], explanation: [" + explanation +"]"; + return "node: [" + executorNode + "], explanation: [" + explanation + "]"; } } @@ -242,7 +244,7 @@ public String toString() { * A record that represents a single running persistent task */ public static class PersistentTask implements Writeable, ToXContent { - private final long id; + private final String id; private final long allocationId; private final String taskName; private final Request request; @@ -253,12 +255,12 @@ public static class PersistentTask implem private final Long allocationIdOnLastStatusUpdate; - public PersistentTask(long id, String taskName, Request request, Assignment assignment) { - this(id, 0L, taskName, request, null, assignment, null); + public PersistentTask(String id, String taskName, Request request, long allocationId, Assignment assignment) { + this(id, allocationId, taskName, request, null, assignment, null); } - public PersistentTask(PersistentTask task, Assignment assignment) { - this(task.id, task.allocationId + 1L, task.taskName, task.request, task.status, + public PersistentTask(PersistentTask task, long allocationId, Assignment assignment) { + this(task.id, allocationId, task.taskName, task.request, task.status, assignment, task.allocationId); } @@ -267,7 +269,7 @@ public PersistentTask(PersistentTask task, Status status) { task.assignment, task.allocationId); } - private PersistentTask(long id, long allocationId, String taskName, Request request, + private PersistentTask(String id, long allocationId, String taskName, Request request, Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; @@ -277,12 +279,12 @@ private PersistentTask(long id, long allocationId, String taskName, Request requ this.assignment = assignment; this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; // Update parent request for starting tasks with correct parent task ID - request.setParentTask("cluster", id); + request.setParentTask("cluster", allocationId); } @SuppressWarnings("unchecked") public PersistentTask(StreamInput in) throws IOException { - id = in.readLong(); + id = in.readString(); allocationId = in.readLong(); taskName = in.readString(); request = (Request) in.readNamedWriteable(PersistentTaskRequest.class); @@ -293,7 +295,7 @@ public PersistentTask(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeLong(id); + out.writeString(id); out.writeLong(allocationId); out.writeString(taskName); out.writeNamedWriteable(request); @@ -308,7 +310,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; PersistentTask that = (PersistentTask) o; - return id == that.id && + return Objects.equals(id, that.id) && allocationId == that.allocationId && Objects.equals(taskName, that.taskName) && Objects.equals(request, that.request) && @@ -328,7 +330,7 @@ public String toString() { return Strings.toString(this); } - public long getId() { + public String getId() { return id; } @@ -421,7 +423,7 @@ public boolean isFragment() { } private static class TaskBuilder { - private long id; + private String id; private long allocationId; private String taskName; private Request request; @@ -429,7 +431,7 @@ private static class TaskBuilder { private Assignment assignment = INITIAL_ASSIGNMENT; private Long allocationIdOnLastStatusUpdate; - public TaskBuilder setId(long id) { + public TaskBuilder setId(String id) { this.id = id; return this; } @@ -477,30 +479,28 @@ public String getWriteableName() { } public PersistentTasksCustomMetaData(StreamInput in) throws IOException { - currentId = in.readLong(); - tasks = in.readMap(StreamInput::readLong, PersistentTask::new); + lastAllocationId = in.readLong(); + tasks = in.readMap(StreamInput::readString, PersistentTask::new); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeLong(currentId); - out.writeMap(tasks, StreamOutput::writeLong, (stream, value) -> { - value.writeTo(stream); - }); + out.writeLong(lastAllocationId); + out.writeMap(tasks, StreamOutput::writeString, (stream, value) -> value.writeTo(stream)); } public static NamedDiff readDiffFrom(StreamInput in) throws IOException { return readDiffFrom(MetaData.Custom.class, TYPE, in); } - public long getCurrentId() { - return currentId; + public long getLastAllocationId() { + return lastAllocationId; } @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.field("current_id", currentId); + builder.field("last_allocation_id", lastAllocationId); builder.startArray("tasks"); for (PersistentTask entry : tasks.values()) { entry.toXContent(builder, params); @@ -518,8 +518,8 @@ public static Builder builder(PersistentTasksCustomMetaData tasks) { } public static class Builder { - private final Map> tasks = new HashMap<>(); - private long currentId; + private final Map> tasks = new HashMap<>(); + private long lastAllocationId; private boolean changed; public Builder() { @@ -528,14 +528,14 @@ public Builder() { public Builder(PersistentTasksCustomMetaData tasksInProgress) { if (tasksInProgress != null) { tasks.putAll(tasksInProgress.tasks); - currentId = tasksInProgress.currentId; + lastAllocationId = tasksInProgress.lastAllocationId; } else { - currentId = 0; + lastAllocationId = 0; } } - private Builder setCurrentId(long currentId) { - this.currentId = currentId; + private Builder setLastAllocationId(long currentId) { + this.lastAllocationId = currentId; return this; } @@ -547,82 +547,79 @@ private Builder setTasks(List - * After the task is added its id can be found by calling {{@link #getCurrentId()}} method. + * After the task is added its id can be found by calling {{@link #getLastAllocationId()}} method. */ - public Builder addTask(String taskName, Request request, Assignment assignment) { + public Builder addTask(String taskId, String taskName, Request request, + Assignment assignment) { changed = true; - currentId++; - tasks.put(currentId, new PersistentTask<>(currentId, taskName, request, assignment)); + PersistentTask previousTask = tasks.put(taskId, new PersistentTask<>(taskId, taskName, request, + getNextAllocationId(), assignment)); + if (previousTask != null) { + throw new ResourceAlreadyExistsException("Trying to override task with id {" + taskId + "}"); + } return this; } /** - * Reassigns the task to another node if the task exist + * Reassigns the task to another node */ - public Builder reassignTask(long taskId, Assignment assignment) { + public Builder reassignTask(String taskId, Assignment assignment) { PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; - tasks.put(taskId, new PersistentTask<>(taskInProgress, assignment)); - } - return this; - } - - /** - * Assigns the task to another node if the task exist and not currently assigned - *

- * The operation is only performed if the task is not currently assigned to any nodes. - */ - @SuppressWarnings("unchecked") - public Builder assignTask(long taskId, - BiFunction executorNodeFunc) { - PersistentTask taskInProgress = (PersistentTask) tasks.get(taskId); - if (taskInProgress != null && taskInProgress.assignment.isAssigned() == false) { // only assign unassigned tasks - Assignment assignment = executorNodeFunc.apply(taskInProgress.taskName, taskInProgress.request); - if (assignment.isAssigned()) { - changed = true; - tasks.put(taskId, new PersistentTask<>(taskInProgress, assignment)); - } + tasks.put(taskId, new PersistentTask<>(taskInProgress, getNextAllocationId(), assignment)); + } else { + throw new ResourceNotFoundException("cannot reassign task with id {" + taskId + "}, the task no longer exits"); } return this; } - /** - * Updates the task status if the task exist + * Updates the task status */ - public Builder updateTaskStatus(long taskId, Status status) { + public Builder updateTaskStatus(String taskId, Status status) { PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; tasks.put(taskId, new PersistentTask<>(taskInProgress, status)); + } else { + throw new ResourceNotFoundException("cannot update task with id {" + taskId + "}, the task no longer exits"); } return this; } /** - * Removes the task if the task exist + * Removes the task */ - public Builder removeTask(long taskId) { + public Builder removeTask(String taskId) { if (tasks.remove(taskId) != null) { changed = true; + } else { + throw new ResourceNotFoundException("cannot remove task with id {" + taskId + "}, the task no longer exits"); } return this; } /** - * Finishes the task if the task exist. - * + * Finishes the task + *

* If the task is marked with removeOnCompletion flag, it is removed from the list, otherwise it is stopped. */ - public Builder finishTask(long taskId) { + public Builder finishTask(String taskId) { PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { changed = true; tasks.remove(taskId); + } else { + throw new ResourceNotFoundException("cannot finish task with id {" + taskId + "}, the task no longer exits"); } return this; } @@ -630,14 +627,14 @@ public Builder finishTask(long taskId) { /** * Checks if the task is currently present in the list */ - public boolean hasTask(long taskId) { + public boolean hasTask(String taskId) { return tasks.containsKey(taskId); } /** * Checks if the task is currently present in the list and has the right allocation id */ - public boolean hasTask(long taskId, long allocationId) { + public boolean hasTask(String taskId, long allocationId) { PersistentTask taskInProgress = tasks.get(taskId); if (taskInProgress != null) { return taskInProgress.getAllocationId() == allocationId; @@ -645,11 +642,8 @@ public boolean hasTask(long taskId, long allocationId) { return false; } - /** - * Returns the id of the last added task - */ - public long getCurrentId() { - return currentId; + Set getCurrentTaskIds() { + return tasks.keySet(); } /** @@ -660,7 +654,7 @@ public boolean isChanged() { } public PersistentTasksCustomMetaData build() { - return new PersistentTasksCustomMetaData(currentId, Collections.unmodifiableMap(tasks)); + return new PersistentTasksCustomMetaData(lastAllocationId, Collections.unmodifiableMap(tasks)); } } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index 1344d9b703c58..66d89aa1f0417 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -48,7 +48,7 @@ * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. */ public class PersistentTasksNodeService extends AbstractComponent implements ClusterStateListener { - private final Map runningTasks = new HashMap<>(); + private final Map runningTasks = new HashMap<>(); private final PersistentTasksService persistentTasksService; private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry; private final TaskManager taskManager; @@ -97,24 +97,24 @@ public void clusterChanged(ClusterChangedEvent event) { if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { // We have some changes let's check if they are related to our node String localNodeId = event.state().getNodes().getLocalNodeId(); - Set notVisitedTasks = new HashSet<>(runningTasks.keySet()); + Set notVisitedTasks = new HashSet<>(runningTasks.keySet()); if (tasks != null) { for (PersistentTask taskInProgress : tasks.tasks()) { if (localNodeId.equals(taskInProgress.getExecutorNode())) { - PersistentTaskId persistentTaskId = new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()); - AllocatedPersistentTask persistentTask = runningTasks.get(persistentTaskId); + Long allocationId = taskInProgress.getAllocationId(); + AllocatedPersistentTask persistentTask = runningTasks.get(allocationId); if (persistentTask == null) { // New task - let's start it startTask(taskInProgress); } else { // The task is still running - notVisitedTasks.remove(persistentTaskId); + notVisitedTasks.remove(allocationId); } } } } - for (PersistentTaskId id : notVisitedTasks) { + for (Long id : notVisitedTasks) { AllocatedPersistentTask task = runningTasks.get(id); if (task.getState() == AllocatedPersistentTask.State.COMPLETED) { // Result was sent to the caller and the caller acknowledged acceptance of the result @@ -139,7 +139,7 @@ private void startTask(PersistentTask void startTask(PersistentTask() { - @Override - public void onResponse(CancelTasksResponse cancelTasksResponse) { - logger.trace("Persistent task with id {} was cancelled", task.getId()); + private void cancelTask(Long allocationId) { + AllocatedPersistentTask task = runningTasks.remove(allocationId); + if (task.markAsCancelled()) { + // Cancel the local task using the task manager + persistentTasksService.sendTaskManagerCancellation(task.getId(), new ActionListener() { + @Override + public void onResponse(CancelTasksResponse cancelTasksResponse) { + logger.trace("Persistent task with id {} was cancelled", task.getId()); - } + } - @Override - public void onFailure(Exception e) { - // There is really nothing we can do in case of failure here - logger.warn((Supplier) () -> new ParameterizedMessage("failed to cancel task {}", task.getPersistentTaskId()), e); - } - }); - } + @Override + public void onFailure(Exception e) { + // There is really nothing we can do in case of failure here + logger.warn((Supplier) () -> new ParameterizedMessage("failed to cancel task {}", task.getPersistentTaskId()), e); + } + }); } } - private static class PersistentTaskId { - private final long id; - private final long allocationId; - - PersistentTaskId(long id, long allocationId) { - this.id = id; - this.allocationId = allocationId; - } - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PersistentTaskId that = (PersistentTaskId) o; - return id == that.id && - allocationId == that.allocationId; - } - - @Override - public int hashCode() { - return Objects.hash(id, allocationId); - } - } public static class Status implements Task.Status { public static final String NAME = "persistent_executor"; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index eb0f9a1a3e8a6..93cd42644afc0 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -58,9 +58,10 @@ public PersistentTasksService(Settings settings, ClusterService clusterService, * Creates the specified persistent task and attempts to assign it to a node. */ @SuppressWarnings("unchecked") - public void startPersistentTask(String taskName, Request request, + public void startPersistentTask(String taskId, String taskName, Request request, ActionListener> listener) { - CreatePersistentTaskAction.Request createPersistentActionRequest = new CreatePersistentTaskAction.Request(taskName, request); + CreatePersistentTaskAction.Request createPersistentActionRequest = + new CreatePersistentTaskAction.Request(taskId, taskName, request); try { client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( o -> listener.onResponse((PersistentTask) o.getTask()), listener::onFailure)); @@ -72,7 +73,7 @@ public void startPersistentTask(String t /** * Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure */ - public void sendCompletionNotification(long taskId, Exception failure, ActionListener> listener) { + public void sendCompletionNotification(String taskId, Exception failure, ActionListener> listener) { CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); try { client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, @@ -103,7 +104,7 @@ void sendTaskManagerCancellation(long taskId, ActionListener> listener) { + void updateStatus(String taskId, long allocationId, Task.Status status, ActionListener> listener) { UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, allocationId, status); try { @@ -117,7 +118,7 @@ void updateStatus(long taskId, long allocationId, Task.Status status, ActionList /** * Cancels if needed and removes a persistent task */ - public void cancelPersistentTask(long taskId, ActionListener> listener) { + public void cancelPersistentTask(String taskId, ActionListener> listener) { RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId); try { client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(o.getTask()), @@ -131,7 +132,7 @@ public void cancelPersistentTask(long taskId, ActionListener> * Checks if the persistent task with giving id (taskId) has the desired state and if it doesn't * waits of it. */ - public void waitForPersistentTaskStatus(long taskId, Predicate> predicate, @Nullable TimeValue timeout, + public void waitForPersistentTaskStatus(String taskId, Predicate> predicate, @Nullable TimeValue timeout, WaitForPersistentTaskStatusListener listener) { ClusterStateObserver stateObserver = new ClusterStateObserver(clusterService, timeout, logger, threadPool.getThreadContext()); if (predicate.test(PersistentTasksCustomMetaData.getTaskWithId(stateObserver.setAndGetObservedState(), taskId))) { diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java index 0209d7745395d..d0a4575a10303 100644 --- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -37,7 +36,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; @@ -67,30 +65,30 @@ public PersistentTaskResponse newResponse() { public static class Request extends MasterNodeRequest { - private long taskId; + private String taskId; public Request() { } - public Request(long taskId) { + public Request(String taskId) { this.taskId = taskId; } - public void setTaskId(long taskId) { + public void setTaskId(String taskId) { this.taskId = taskId; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - taskId = in.readLong(); + taskId = in.readString(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeLong(taskId); + out.writeString(taskId); } @Override @@ -103,7 +101,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return taskId == request.taskId; + return Objects.equals(taskId, request.taskId); } @Override @@ -119,7 +117,7 @@ protected RequestBuilder(ElasticsearchClient client, RemovePersistentTaskAction super(client, action, new Request()); } - public final RequestBuilder setTaskId(long taskId) { + public final RequestBuilder setTaskId(String taskId) { request.setTaskId(taskId); return this; } diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java index 772d200b24eb8..8c1de889e0d48 100644 --- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -68,7 +68,7 @@ public PersistentTaskResponse newResponse() { public static class Request extends MasterNodeRequest { - private long taskId; + private String taskId; private long allocationId; private Task.Status status; @@ -76,13 +76,13 @@ public Request() { } - public Request(long taskId, long allocationId, Task.Status status) { + public Request(String taskId, long allocationId, Task.Status status) { this.taskId = taskId; this.allocationId = allocationId; this.status = status; } - public void setTaskId(long taskId) { + public void setTaskId(String taskId) { this.taskId = taskId; } @@ -97,7 +97,7 @@ public void setStatus(Task.Status status) { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - taskId = in.readLong(); + taskId = in.readString(); allocationId = in.readLong(); status = in.readOptionalNamedWriteable(Task.Status.class); } @@ -105,7 +105,7 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeLong(taskId); + out.writeString(taskId); out.writeLong(allocationId); out.writeOptionalNamedWriteable(status); } @@ -120,7 +120,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return taskId == request.taskId && allocationId == request.allocationId && + return Objects.equals(taskId, request.taskId) && allocationId == request.allocationId && Objects.equals(status, request.status); } @@ -137,7 +137,7 @@ protected RequestBuilder(ElasticsearchClient client, UpdatePersistentTaskStatusA super(client, action, new Request()); } - public final RequestBuilder setTaskId(long taskId) { + public final RequestBuilder setTaskId(String taskId) { request.setTaskId(taskId); return this; } diff --git a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java index db263428cba4c..2ce82c1e79941 100644 --- a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java @@ -18,14 +18,16 @@ */ package org.elasticsearch.persistent; -import org.elasticsearch.persistent.RemovePersistentTaskAction.Request; import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.RemovePersistentTaskAction.Request; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; public class CancelPersistentTaskRequestTests extends AbstractStreamableTestCase { @Override protected Request createTestInstance() { - return new Request(randomLong()); + return new Request(randomAsciiOfLength(10)); } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index aea9849ba58dd..77d8a0a8c2103 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -403,11 +404,11 @@ private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuil MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, Assignment assignment, String param) { return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, - tasks.addTask(randomAlphaOfLength(10), new TestRequest(param), assignment).build())); + tasks.addTask(UUIDs.base64UUID(), randomAlphaOfLength(10), new TestRequest(param), assignment).build())); } private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node) { - tasks.addTask(action, new TestRequest(param), new Assignment(node, "explanation: " + action)); + tasks.addTask(UUIDs.base64UUID(), action, new TestRequest(param), new Assignment(node, "explanation: " + action)); } private DiscoveryNode newNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index d20042ba601a0..e66b0ca1af6fc 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -18,11 +18,13 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaData.Custom; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.io.stream.Writeable; @@ -57,11 +59,12 @@ protected PersistentTasksCustomMetaData createTestInstance() { int numberOfTasks = randomInt(10); PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); for (int i = 0; i < numberOfTasks; i++) { - tasks.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), + String taskId = UUIDs.base64UUID(); + tasks.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), randomAssignment()); if (randomBoolean()) { // From time to time update status - tasks.updateTaskStatus(tasks.getCurrentId(), new Status(randomAlphaOfLength(10))); + tasks.updateTaskStatus(taskId, new Status(randomAlphaOfLength(10))); } } return tasks.build(); @@ -84,31 +87,30 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { @Override protected Custom makeTestChanges(Custom testInstance) { - PersistentTasksCustomMetaData tasksInProgress = (PersistentTasksCustomMetaData) testInstance; - Builder builder = new Builder(); + Builder builder = new Builder((PersistentTasksCustomMetaData) testInstance); switch (randomInt(3)) { case 0: addRandomTask(builder); break; case 1: - if (tasksInProgress.tasks().isEmpty()) { + if (builder.getCurrentTaskIds().isEmpty()) { addRandomTask(builder); } else { - builder.reassignTask(pickRandomTask(tasksInProgress), randomAssignment()); + builder.reassignTask(pickRandomTask(builder), randomAssignment()); } break; case 2: - if (tasksInProgress.tasks().isEmpty()) { + if (builder.getCurrentTaskIds().isEmpty()) { addRandomTask(builder); } else { - builder.updateTaskStatus(pickRandomTask(tasksInProgress), randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); + builder.updateTaskStatus(pickRandomTask(builder), randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); } break; case 3: - if (tasksInProgress.tasks().isEmpty()) { + if (builder.getCurrentTaskIds().isEmpty()) { addRandomTask(builder); } else { - builder.removeTask(pickRandomTask(tasksInProgress)); + builder.removeTask(pickRandomTask(builder)); } break; } @@ -147,13 +149,14 @@ protected XContentBuilder toXContent(Custom instance, XContentType contentType, return builder; } - private Builder addRandomTask(Builder builder) { - builder.addTask(TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), randomAssignment()); - return builder; + private String addRandomTask(Builder builder) { + String taskId = UUIDs.base64UUID(); + builder.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), randomAssignment()); + return taskId; } - private long pickRandomTask(PersistentTasksCustomMetaData testInstance) { - return randomFrom(new ArrayList<>(testInstance.tasks())).getId(); + private String pickRandomTask(PersistentTasksCustomMetaData.Builder testInstance) { + return randomFrom(new ArrayList<>(testInstance.getCurrentTaskIds())); } @Override @@ -202,7 +205,7 @@ public void testSerializationContext() throws Exception { public void testBuilder() { PersistentTasksCustomMetaData persistentTasks = null; - long lastKnownTask = -1; + String lastKnownTask = ""; for (int i = 0; i < randomIntBetween(10, 100); i++) { final Builder builder; if (randomBoolean()) { @@ -212,54 +215,46 @@ public void testBuilder() { } boolean changed = false; for (int j = 0; j < randomIntBetween(1, 10); j++) { - switch (randomInt(5)) { + switch (randomInt(4)) { case 0: - lastKnownTask = addRandomTask(builder).getCurrentId(); + lastKnownTask = addRandomTask(builder); changed = true; break; case 1: if (builder.hasTask(lastKnownTask)) { changed = true; + builder.reassignTask(lastKnownTask, randomAssignment()); + } else { + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.reassignTask(fLastKnownTask, randomAssignment())); } - builder.reassignTask(lastKnownTask, randomAssignment()); break; case 2: if (builder.hasTask(lastKnownTask)) { - PersistentTask task = builder.build().getTask(lastKnownTask); - if (randomBoolean()) { - // Trying to reassign to the same node - builder.assignTask(lastKnownTask, (s, request) -> task.getAssignment()); - } else { - // Trying to reassign to a different node - Assignment randomAssignment = randomAssignment(); - builder.assignTask(lastKnownTask, (s, request) -> randomAssignment); - // should change if the task was unassigned and was reassigned to a different node or started - if ((task.isAssigned() == false && randomAssignment.isAssigned())) { - changed = true; - } - } + changed = true; + builder.updateTaskStatus(lastKnownTask, randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); } else { - // task doesn't exist - shouldn't change - builder.assignTask(lastKnownTask, (s, request) -> randomAssignment()); + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.updateTaskStatus(fLastKnownTask, null)); } break; case 3: if (builder.hasTask(lastKnownTask)) { changed = true; + builder.removeTask(lastKnownTask); + } else { + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.removeTask(fLastKnownTask)); } - builder.updateTaskStatus(lastKnownTask, randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); break; case 4: if (builder.hasTask(lastKnownTask)) { changed = true; + builder.finishTask(lastKnownTask); + } else { + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.finishTask(fLastKnownTask)); } - builder.removeTask(lastKnownTask); - break; - case 5: - if (builder.hasTask(lastKnownTask)) { - changed = true; - } - builder.finishTask(lastKnownTask); break; } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index fc6b3abee7766..b68464636f541 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.persistent; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; @@ -57,17 +58,18 @@ protected boolean ignoreExternalCluster() { public void testFullClusterRestart() throws Exception { PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class); int numberOfTasks = randomIntBetween(1, 10); - long[] taskIds = new long[numberOfTasks]; + String[] taskIds = new String[numberOfTasks]; List>> futures = new ArrayList<>(numberOfTasks); for (int i = 0; i < numberOfTasks; i++) { PlainActionFuture> future = new PlainActionFuture<>(); futures.add(future); - service.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + taskIds[i] = UUIDs.base64UUID(); + service.startPersistentTask(taskIds[i], TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); } for (int i = 0; i < numberOfTasks; i++) { - taskIds[i] = futures.get(i).get().getId(); + assertThat(futures.get(i).get().getId(), equalTo(taskIds[i])); } PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 399fd3053030b..40cecfb293627 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -19,8 +19,10 @@ package org.elasticsearch.persistent; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.plugins.Plugin; @@ -75,8 +77,8 @@ public static class WaitForPersistentTaskStatusFuture> future = new PlainActionFuture<>(); - persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - long taskId = future.get().getId(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long allocationId = future.get().getAllocationId(); assertBusy(() -> { // Wait for the task to start assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() @@ -86,7 +88,7 @@ public void testPersistentActionFailure() throws Exception { .get().getTasks().get(0); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); // Verifying parent - assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(taskId)); + assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(allocationId)); assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); logger.info("Failing the running task"); @@ -105,8 +107,8 @@ public void testPersistentActionFailure() throws Exception { public void testPersistentActionCompletion() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); PlainActionFuture> future = new PlainActionFuture<>(); - persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - long taskId = future.get().getId(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + long taskId = future.get().getAllocationId(); assertBusy(() -> { // Wait for the task to start assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() @@ -126,8 +128,8 @@ public void testPersistentActionWithNoAvailableNode() throws Exception { PlainActionFuture> future = new PlainActionFuture<>(); TestRequest testRequest = new TestRequest("Blah"); testRequest.setExecutorNodeAttr("test"); - persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, testRequest, future); - long taskId = future.get().getId(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, testRequest, future); + String taskId = future.get().getId(); Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); String newNode = internalCluster().startNode(nodeSettings); @@ -160,8 +162,8 @@ public void testPersistentActionWithNoAvailableNode() throws Exception { public void testPersistentActionStatusUpdate() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); PlainActionFuture> future = new PlainActionFuture<>(); - persistentTasksService.startPersistentTask(TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - long taskId = future.get().getId(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + String taskId = future.get().getId(); assertBusy(() -> { // Wait for the task to start @@ -215,6 +217,38 @@ public void testPersistentActionStatusUpdate() throws Exception { assertThat(future2.get(), nullValue()); } + public void testCreatePersistentTaskWithDuplicateId() throws Exception { + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PlainActionFuture> future = new PlainActionFuture<>(); + String taskId = UUIDs.base64UUID(); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + future.get(); + + PlainActionFuture> future2 = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future2); + assertThrows(future2, ResourceAlreadyExistsException.class); + + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); + }); + + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .get().getTasks().get(0); + + logger.info("Completing the running task"); + // Fail the running task and make sure it restarts properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); + assertBusy(() -> { + // Wait for the task to disappear completely + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), + empty()); + }); + } private void stopOrCancelTask(TaskId taskId) { if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java index 8d1e4796ac1b8..eb18e4fdf8652 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.test.AbstractStreamableTestCase; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; @@ -32,9 +33,9 @@ public class PersistentTasksExecutorResponseTests extends AbstractStreamableTest protected PersistentTaskResponse createTestInstance() { if (randomBoolean()) { return new PersistentTaskResponse( - new PersistentTask(randomLong(), randomAsciiOfLength(10), + new PersistentTask(UUIDs.base64UUID(), randomAsciiOfLength(10), new TestPersistentTasksPlugin.TestRequest("test"), - PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT)); + randomLong(), PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT)); } else { return new PersistentTaskResponse(null); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index c35432deddd2d..8d1960bb8054d 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -90,11 +90,11 @@ public void testStartTask() throws Exception { boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { - tasks.addTask("test_action", new TestRequest("other_" + i), + tasks.addTask(UUIDs.base64UUID(), "test_action", new TestRequest("other_" + i), new Assignment("other_node_" + randomInt(nonLocalNodesCount), "test assignment on other node")); if (added == false && randomBoolean()) { added = true; - tasks.addTask("test", new TestRequest("this_param"), + tasks.addTask(UUIDs.base64UUID(), "test", new TestRequest("this_param"), new Assignment("this_node", "test assignment on this node")); } } @@ -132,8 +132,8 @@ public void testStartTask() throws Exception { // Finish both tasks executor.get(0).task.markAsFailed(new RuntimeException()); executor.get(1).task.markAsCompleted(); - long failedTaskId = executor.get(0).task.getParentTaskId().getId(); - long finishedTaskId = executor.get(1).task.getParentTaskId().getId(); + String failedTaskId = executor.get(0).task.getPersistentTaskId(); + String finishedTaskId = executor.get(1).task.getPersistentTaskId(); executor.clear(); // Add task on some other node @@ -172,7 +172,7 @@ public void sendTaskManagerCancellation(long taskId, ActionListener> listener) { + public void sendCompletionNotification(String taskId, Exception failure, ActionListener> listener) { fail("Shouldn't be called during Cluster State cancellation"); } }; @@ -198,8 +198,8 @@ public void sendCompletionNotification(long taskId, Exception failure, ActionLis // Check the the task is know to the task manager assertThat(taskManager.getTasks().size(), equalTo(1)); - Task runningTask = taskManager.getTasks().values().iterator().next(); - long persistentId = runningTask.getParentTaskId().getId(); + AllocatedPersistentTask runningTask = (AllocatedPersistentTask)taskManager.getTasks().values().iterator().next(); + String persistentId = runningTask.getPersistentTaskId(); long localId = runningTask.getId(); // Make sure it returns correct status Task.Status status = runningTask.getStatus(); @@ -241,10 +241,10 @@ private ClusterState addTask(ClusterStat PersistentTasksCustomMetaData.Builder builder = PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, - builder.addTask(action, request, new Assignment(node, "test assignment")).build())).build(); + builder.addTask(UUIDs.base64UUID(), action, request, new Assignment(node, "test assignment")).build())).build(); } - private ClusterState reallocateTask(ClusterState state, long taskId, String node) { + private ClusterState reallocateTask(ClusterState state, String taskId, String node) { PersistentTasksCustomMetaData.Builder builder = PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); assertTrue(builder.hasTask(taskId)); @@ -252,7 +252,7 @@ private ClusterState reallocateTask(ClusterState state, long taskId, String node builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build(); } - private ClusterState removeTask(ClusterState state, long taskId) { + private ClusterState removeTask(ClusterState state, String taskId) { PersistentTasksCustomMetaData.Builder builder = PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); assertTrue(builder.hasTask(taskId)); diff --git a/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java index 59883dc8d4e40..3fc8f80490359 100644 --- a/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java @@ -18,14 +18,14 @@ */ package org.elasticsearch.persistent; -import org.elasticsearch.persistent.CompletionPersistentTaskAction.Request; import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.CompletionPersistentTaskAction.Request; public class RestartPersistentTaskRequestTests extends AbstractStreamableTestCase { @Override protected Request createTestInstance() { - return new Request(randomLong(), null); + return new Request(randomAlphaOfLength(10), null); } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java index fa425454a0993..b4e7fab62ec82 100644 --- a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.persistent.CreatePersistentTaskAction.Request; @@ -41,7 +42,7 @@ protected Request createTestInstance() { if (randomBoolean()) { testRequest.setExecutorNodeAttr(randomAlphaOfLengthBetween(1, 20)); } - return new Request(randomAlphaOfLengthBetween(1, 20), new TestRequest()); + return new Request(UUIDs.base64UUID(), randomAlphaOfLengthBetween(1, 20), new TestRequest()); } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java index bc80c0d7211f3..fab4e2e2a758f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractStreamableTestCase; @@ -30,7 +31,7 @@ public class UpdatePersistentTaskRequestTests extends AbstractStreamableTestCase @Override protected Request createTestInstance() { - return new Request(randomLong(), randomLong(), new Status(randomAlphaOfLength(10))); + return new Request(UUIDs.base64UUID(), randomLong(), new Status(randomAlphaOfLength(10))); } @Override From abd9ae399ccfec19b04d0c5e10497c5a5e1b112f Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 12 Apr 2017 09:58:15 -0400 Subject: [PATCH 28/50] Persistent Tasks: PersistentTaskRequest -> PersistTaskParams (#1057) Removes the last pieces of ActionRequest from PersistentTaskRequest and renames it into PersistTaskParams, which is now just an interface that extends NamedWriteable and ToXContent. --- .../NodePersistentTasksExecutor.java | 11 +- ...Request.java => PersistentTaskParams.java} | 13 +-- .../PersistentTasksClusterService.java | 32 +++--- .../PersistentTasksCustomMetaData.java | 106 +++++++++--------- .../persistent/PersistentTasksExecutor.java | 34 ++++-- .../PersistentTasksExecutorRegistry.java | 4 +- .../PersistentTasksNodeService.java | 30 ++++- .../persistent/PersistentTasksService.java | 16 +-- ...on.java => StartPersistentTaskAction.java} | 56 ++++----- .../PersistentTasksClusterServiceTests.java | 28 ++--- .../PersistentTasksCustomMetaDataTests.java | 16 +-- .../PersistentTasksExecutorFullRestartIT.java | 9 +- .../persistent/PersistentTasksExecutorIT.java | 46 ++++---- .../PersistentTasksExecutorResponseTests.java | 7 +- .../PersistentTasksNodeServiceTests.java | 51 +++++---- .../StartPersistentActionRequestTests.java | 26 +++-- .../persistent/TestPersistentTasksPlugin.java | 66 +++++------ 17 files changed, 300 insertions(+), 251 deletions(-) rename server/src/main/java/org/elasticsearch/persistent/{PersistentTaskRequest.java => PersistentTaskParams.java} (66%) rename server/src/main/java/org/elasticsearch/persistent/{CreatePersistentTaskAction.java => StartPersistentTaskAction.java} (82%) diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index 4e4c0692a95eb..7efbc92b27809 100644 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.threadpool.ThreadPool; @@ -33,10 +34,10 @@ public NodePersistentTasksExecutor(ThreadPool threadPool) { this.threadPool = threadPool; } - public void executeTask(Request request, - AllocatedPersistentTask task, - PersistentTasksExecutor action) { - threadPool.executor(action.getExecutor()).execute(new AbstractRunnable() { + public void executeTask(@Nullable Params params, + AllocatedPersistentTask task, + PersistentTasksExecutor executor) { + threadPool.executor(executor.getExecutor()).execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { task.markAsFailed(e); @@ -46,7 +47,7 @@ public void onFailure(Exception e) { @Override protected void doRun() throws Exception { try { - action.nodeOperation(task, request); + executor.nodeOperation(task, params); } catch (Exception ex) { task.markAsFailed(ex); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java similarity index 66% rename from server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java rename to server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java index b07bfbfc79df5..ecf5c7c318978 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskRequest.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java @@ -16,20 +16,15 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.persistent; -import org.elasticsearch.action.ActionRequest; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskId; /** - * Base class for a request for a persistent task + * Parameters used to start persistent task */ -public abstract class PersistentTaskRequest extends ActionRequest implements NamedWriteable, ToXContent { - @Override - public Task createTask(long id, String type, String action, TaskId parentTaskId) { - return new AllocatedPersistentTask(id, type, action, getDescription(), parentTaskId); - } +public interface PersistentTaskParams extends NamedWriteable, ToXContent { + } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index fb14922ae3eb7..57b370398f1fc 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; @@ -58,11 +59,11 @@ public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorR * Creates a new persistent task on master node * * @param action the action name - * @param request request + * @param params params * @param listener the listener that will be called when task is started */ - public void createPersistentTask(String taskId, String action, Request request, - ActionListener> listener) { + public void createPersistentTask(String taskId, String action, @Nullable Params params, + ActionListener> listener) { clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -70,10 +71,10 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (builder.hasTask(taskId)) { throw new ResourceAlreadyExistsException("task with id {" + taskId + "} already exist"); } - validate(action, clusterService.state(), request); + validate(action, clusterService.state(), params); final Assignment assignment; - assignment = getAssignement(action, currentState, request); - return update(currentState, builder.addTask(taskId, action, request, assignment)); + assignment = getAssignement(action, currentState, params); + return update(currentState, builder.addTask(taskId, action, params, assignment)); } @Override @@ -205,14 +206,15 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } - private Assignment getAssignement(String taskName, ClusterState currentState, Request request) { - PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); - return persistentTasksExecutor.getAssignment(request, currentState); + private Assignment getAssignement(String taskName, ClusterState currentState, + @Nullable Params params) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + return persistentTasksExecutor.getAssignment(params, currentState); } - private void validate(String taskName, ClusterState currentState, Request request) { - PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); - persistentTasksExecutor.validate(request, currentState); + private void validate(String taskName, ClusterState currentState, @Nullable Params params) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + persistentTasksExecutor.validate(params, currentState); } @Override @@ -229,7 +231,7 @@ public void clusterChanged(ClusterChangedEvent event) { } interface ExecutorNodeDecider { - Assignment getAssignment(String action, ClusterState currentState, Request request); + Assignment getAssignment(String action, ClusterState currentState, Params params); } static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { @@ -245,7 +247,7 @@ static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecid if (taskInProgress.needsReassignment(event.state().nodes())) { // there is an unassigned task or task with a disappeared node - we need to try assigning it if (Objects.equals(taskInProgress.getAssignment(), - decider.getAssignment(taskInProgress.getTaskName(), event.state(), taskInProgress.getRequest())) == false) { + decider.getAssignment(taskInProgress.getTaskName(), event.state(), taskInProgress.getParams())) == false) { // it looks like a assignment for at least one task is possible - let's trigger reassignment reassignmentRequired = true; break; @@ -290,7 +292,7 @@ static ClusterState reassignTasks(ClusterState currentState, Logger logger, Exec for (PersistentTask task : tasks.tasks()) { if (task.needsReassignment(nodes)) { // there is an unassigned task - we need to try assigning it - Assignment assignment = decider.getAssignment(task.getTaskName(), clusterState, task.getRequest()); + Assignment assignment = decider.getAssignment(task.getTaskName(), clusterState, task.getParams()); if (Objects.equals(assignment, task.getAssignment()) == false) { logger.trace("reassigning task {} from node {} to node {}", task.getId(), task.getAssignment().getExecutorNode(), assignment.getExecutorNode()); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 70dce599cbcec..18d8b00fea5a6 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -76,14 +76,14 @@ public PersistentTasksCustomMetaData(long lastAllocationId, Map PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new); - private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = + private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = new ObjectParser<>("tasks", TaskBuilder::new); public static final ConstructingObjectParser ASSIGNMENT_PARSER = new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); - private static final NamedObjectParser REQUEST_PARSER = - (XContentParser p, Void c, String name) -> p.namedObject(PersistentTaskRequest.class, name, null); + private static final NamedObjectParser PARAMS_PARSER = + (XContentParser p, Void c, String name) -> p.namedObject(PersistentTaskParams.class, name, null); private static final NamedObjectParser STATUS_PARSER = (XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null); @@ -101,15 +101,15 @@ public PersistentTasksCustomMetaData(long lastAllocationId, Map taskBuilder, List objects) -> { + (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { - throw new IllegalArgumentException("only one request per task is allowed"); + throw new IllegalArgumentException("only one params per task is allowed"); } - taskBuilder.setRequest(objects.get(0)); - }, REQUEST_PARSER, new ParseField("request")); + taskBuilder.setParams(objects.get(0)); + }, PARAMS_PARSER, new ParseField("params")); PERSISTENT_TASK_PARSER.declareNamedObjects( - (TaskBuilder taskBuilder, List objects) -> { + (TaskBuilder taskBuilder, List objects) -> { if (objects.size() != 1) { throw new IllegalArgumentException("only one status per task is allowed"); } @@ -186,10 +186,10 @@ public static PersistentTasksCustomMetaData fromXContent(XContentParser parser) } @SuppressWarnings("unchecked") - public static PersistentTask getTaskWithId(ClusterState clusterState, String taskId) { + public static PersistentTask getTaskWithId(ClusterState clusterState, String taskId) { PersistentTasksCustomMetaData tasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); if (tasks != null) { - return (PersistentTask) tasks.getTask(taskId); + return (PersistentTask) tasks.getTask(taskId); } return null; } @@ -243,11 +243,12 @@ public String toString() { /** * A record that represents a single running persistent task */ - public static class PersistentTask implements Writeable, ToXContent { + public static class PersistentTask implements Writeable, ToXContent { private final String id; private final long allocationId; private final String taskName; - private final Request request; + @Nullable + private final Params params; @Nullable private final Status status; private final Assignment assignment; @@ -255,31 +256,29 @@ public static class PersistentTask implem private final Long allocationIdOnLastStatusUpdate; - public PersistentTask(String id, String taskName, Request request, long allocationId, Assignment assignment) { - this(id, allocationId, taskName, request, null, assignment, null); + public PersistentTask(String id, String taskName, Params params, long allocationId, Assignment assignment) { + this(id, allocationId, taskName, params, null, assignment, null); } - public PersistentTask(PersistentTask task, long allocationId, Assignment assignment) { - this(task.id, allocationId, task.taskName, task.request, task.status, + public PersistentTask(PersistentTask task, long allocationId, Assignment assignment) { + this(task.id, allocationId, task.taskName, task.params, task.status, assignment, task.allocationId); } - public PersistentTask(PersistentTask task, Status status) { - this(task.id, task.allocationId, task.taskName, task.request, status, + public PersistentTask(PersistentTask task, Status status) { + this(task.id, task.allocationId, task.taskName, task.params, status, task.assignment, task.allocationId); } - private PersistentTask(String id, long allocationId, String taskName, Request request, + private PersistentTask(String id, long allocationId, String taskName, Params params, Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; this.taskName = taskName; - this.request = request; + this.params = params; this.status = status; this.assignment = assignment; this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; - // Update parent request for starting tasks with correct parent task ID - request.setParentTask("cluster", allocationId); } @SuppressWarnings("unchecked") @@ -287,7 +286,7 @@ public PersistentTask(StreamInput in) throws IOException { id = in.readString(); allocationId = in.readLong(); taskName = in.readString(); - request = (Request) in.readNamedWriteable(PersistentTaskRequest.class); + params = (Params) in.readOptionalNamedWriteable(PersistentTaskParams.class); status = in.readOptionalNamedWriteable(Task.Status.class); assignment = new Assignment(in.readOptionalString(), in.readString()); allocationIdOnLastStatusUpdate = in.readOptionalLong(); @@ -298,7 +297,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeLong(allocationId); out.writeString(taskName); - out.writeNamedWriteable(request); + out.writeOptionalNamedWriteable(params); out.writeOptionalNamedWriteable(status); out.writeOptionalString(assignment.executorNode); out.writeString(assignment.explanation); @@ -313,7 +312,7 @@ public boolean equals(Object o) { return Objects.equals(id, that.id) && allocationId == that.allocationId && Objects.equals(taskName, that.taskName) && - Objects.equals(request, that.request) && + Objects.equals(params, that.params) && Objects.equals(status, that.status) && Objects.equals(assignment, that.assignment) && Objects.equals(allocationIdOnLastStatusUpdate, that.allocationIdOnLastStatusUpdate); @@ -321,7 +320,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(id, allocationId, taskName, request, status, assignment, + return Objects.hash(id, allocationId, taskName, params, status, assignment, allocationIdOnLastStatusUpdate); } @@ -342,8 +341,9 @@ public String getTaskName() { return taskName; } - public Request getRequest() { - return request; + @Nullable + public Params getParams() { + return params; } @Nullable @@ -380,25 +380,27 @@ public boolean isCurrentStatus() { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params xParams) throws IOException { builder.startObject(); { builder.field("id", id); builder.field("name", taskName); - builder.startObject("request"); - { - builder.field(request.getWriteableName(), request, params); + if (params != null) { + builder.startObject("params"); + { + builder.field(params.getWriteableName(), params, xParams); + } + builder.endObject(); } - builder.endObject(); if (status != null) { builder.startObject("status"); { - builder.field(status.getWriteableName(), status, params); + builder.field(status.getWriteableName(), status, xParams); } builder.endObject(); } - if (API_CONTEXT.equals(params.param(MetaData.CONTEXT_MODE_PARAM, API_CONTEXT))) { + if (API_CONTEXT.equals(xParams.param(MetaData.CONTEXT_MODE_PARAM, API_CONTEXT))) { // These are transient values that shouldn't be persisted to gateway cluster state or snapshot builder.field("allocation_id", allocationId); builder.startObject("assignment"); @@ -422,53 +424,53 @@ public boolean isFragment() { } } - private static class TaskBuilder { + private static class TaskBuilder { private String id; private long allocationId; private String taskName; - private Request request; + private Params params; private Status status; private Assignment assignment = INITIAL_ASSIGNMENT; private Long allocationIdOnLastStatusUpdate; - public TaskBuilder setId(String id) { + public TaskBuilder setId(String id) { this.id = id; return this; } - public TaskBuilder setAllocationId(long allocationId) { + public TaskBuilder setAllocationId(long allocationId) { this.allocationId = allocationId; return this; } - public TaskBuilder setTaskName(String taskName) { + public TaskBuilder setTaskName(String taskName) { this.taskName = taskName; return this; } - public TaskBuilder setRequest(Request request) { - this.request = request; + public TaskBuilder setParams(Params params) { + this.params = params; return this; } - public TaskBuilder setStatus(Status status) { + public TaskBuilder setStatus(Status status) { this.status = status; return this; } - public TaskBuilder setAssignment(Assignment assignment) { + public TaskBuilder setAssignment(Assignment assignment) { this.assignment = assignment; return this; } - public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdOnLastStatusUpdate) { + public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdOnLastStatusUpdate) { this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; return this; } - public PersistentTask build() { - return new PersistentTask<>(id, allocationId, taskName, request, status, + public PersistentTask build() { + return new PersistentTask<>(id, allocationId, taskName, params, status, assignment, allocationIdOnLastStatusUpdate); } } @@ -499,7 +501,7 @@ public long getLastAllocationId() { @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("last_allocation_id", lastAllocationId); builder.startArray("tasks"); for (PersistentTask entry : tasks.values()) { @@ -539,7 +541,7 @@ private Builder setLastAllocationId(long currentId) { return this; } - private Builder setTasks(List> tasks) { + private Builder setTasks(List> tasks) { for (TaskBuilder builder : tasks) { PersistentTask task = builder.build(); this.tasks.put(task.getId(), task); @@ -557,10 +559,10 @@ private long getNextAllocationId() { *

* After the task is added its id can be found by calling {{@link #getLastAllocationId()}} method. */ - public Builder addTask(String taskId, String taskName, Request request, - Assignment assignment) { + public Builder addTask(String taskId, String taskName, Params params, + Assignment assignment) { changed = true; - PersistentTask previousTask = tasks.put(taskId, new PersistentTask<>(taskId, taskName, request, + PersistentTask previousTask = tasks.put(taskId, new PersistentTask<>(taskId, taskName, params, getNextAllocationId(), assignment)); if (previousTask != null) { throw new ResourceAlreadyExistsException("Trying to override task with id {" + taskId + "}"); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 93be9d068415b..d8f5d93126a60 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -19,13 +19,14 @@ package org.elasticsearch.persistent; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.util.function.Predicate; @@ -33,7 +34,7 @@ * An executor of tasks that can survive restart of requesting or executing node. * These tasks are using cluster state rather than only transport service to send requests and responses. */ -public abstract class PersistentTasksExecutor extends AbstractComponent { +public abstract class PersistentTasksExecutor extends AbstractComponent { private final String executor; private final String taskName; @@ -51,11 +52,11 @@ public String getTaskName() { public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment"); /** - * Returns the node id where the request has to be executed, + * Returns the node id where the params has to be executed, *

* The default implementation returns the least loaded data node */ - public Assignment getAssignment(Request request, ClusterState clusterState) { + public Assignment getAssignment(Params params, ClusterState clusterState) { DiscoveryNode discoveryNode = selectLeastLoadedNode(clusterState, DiscoveryNode::isDataNode); if (discoveryNode == null) { return NO_NODE_FOUND; @@ -88,21 +89,36 @@ protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predica } /** - * Checks the current cluster state for compatibility with the request + * Checks the current cluster state for compatibility with the params *

- * Throws an exception if the supplied request cannot be executed on the cluster in the current state. + * Throws an exception if the supplied params cannot be executed on the cluster in the current state. */ - public void validate(Request request, ClusterState clusterState) { + public void validate(Params params, ClusterState clusterState) { } + /** + * Creates a AllocatedPersistentTask for communicating with task manager + */ + protected AllocatedPersistentTask createTask(long id, String type, String action, TaskId parentTaskId, + PersistentTask taskInProgress) { + return new AllocatedPersistentTask(id, type, action, getDescription(taskInProgress), parentTaskId); + } + + /** + * Returns task description that will be available via task manager + */ + protected String getDescription(PersistentTask taskInProgress) { + return "id=" + taskInProgress.getId(); + } + /** * This operation will be executed on the executor node. *

* NOTE: The nodeOperation has to throws an exception, trigger task.markAsCompleted() or task.completeAndNotifyIfNeeded() methods to * indicate that the persistent task has finished. */ - protected abstract void nodeOperation(AllocatedPersistentTask task, Request request); + protected abstract void nodeOperation(AllocatedPersistentTask task, @Nullable Params params); public String getExecutor() { return executor; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java index e506f0ad748e3..2ac57e074b7bf 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java @@ -44,8 +44,8 @@ public PersistentTasksExecutorRegistry(Settings settings, Collection PersistentTasksExecutor getPersistentTaskExecutorSafe(String taskName) { - PersistentTasksExecutor executor = (PersistentTasksExecutor) taskExecutors.get(taskName); + public PersistentTasksExecutor getPersistentTaskExecutorSafe(String taskName) { + PersistentTasksExecutor executor = (PersistentTasksExecutor) taskExecutors.get(taskName); if (executor == null) { throw new IllegalStateException("Unknown persistent executor [" + taskName + "]"); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index 66d89aa1f0417..4222c49107f76 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -31,6 +31,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskAwareRequest; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; @@ -131,16 +133,36 @@ public void clusterChanged(ClusterChangedEvent event) { } - private void startTask(PersistentTask taskInProgress) { - PersistentTasksExecutor action = persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName()); + private void startTask(PersistentTask taskInProgress) { + PersistentTasksExecutor executor = + persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName()); + + TaskAwareRequest request = new TaskAwareRequest() { + TaskId parentTaskId = new TaskId("cluster", taskInProgress.getAllocationId()); + + @Override + public void setParentTask(TaskId taskId) { + throw new UnsupportedOperationException("parent task if for persistent tasks shouldn't change"); + } + + @Override + public TaskId getParentTask() { + return parentTaskId; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return executor.createTask(id, type, action, parentTaskId, taskInProgress); + } + }; AllocatedPersistentTask task = (AllocatedPersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]", - taskInProgress.getRequest()); + request); boolean processed = false; try { task.init(persistentTasksService, taskManager, logger, taskInProgress.getId(), taskInProgress.getAllocationId()); try { runningTasks.put(taskInProgress.getAllocationId(), task); - nodePersistentTasksExecutor.executeTask(taskInProgress.getRequest(), task, action); + nodePersistentTasksExecutor.executeTask(taskInProgress.getParams(), task, executor); } catch (Exception e) { // Submit task failure task.markAsFailed(e); diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 93cd42644afc0..d7788ae5af9a9 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -58,13 +58,13 @@ public PersistentTasksService(Settings settings, ClusterService clusterService, * Creates the specified persistent task and attempts to assign it to a node. */ @SuppressWarnings("unchecked") - public void startPersistentTask(String taskId, String taskName, Request request, - ActionListener> listener) { - CreatePersistentTaskAction.Request createPersistentActionRequest = - new CreatePersistentTaskAction.Request(taskId, taskName, request); + public void startPersistentTask(String taskId, String taskName, @Nullable Params params, + ActionListener> listener) { + StartPersistentTaskAction.Request createPersistentActionRequest = + new StartPersistentTaskAction.Request(taskId, taskName, params); try { - client.execute(CreatePersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( - o -> listener.onResponse((PersistentTask) o.getTask()), listener::onFailure)); + client.execute(StartPersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( + o -> listener.onResponse((PersistentTask) o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); } @@ -183,8 +183,8 @@ public void onTimeout(TimeValue timeout) { } } - public interface WaitForPersistentTaskStatusListener - extends ActionListener> { + public interface WaitForPersistentTaskStatusListener + extends ActionListener> { default void onTimeout(TimeValue timeout) { onFailure(new IllegalStateException("timed out after " + timeout)); } diff --git a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java similarity index 82% rename from server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java rename to server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 6c2b8311d6192..40d36b9a439d6 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CreatePersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -11,6 +11,10 @@ * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.elasticsearch.persistent; @@ -27,6 +31,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -43,14 +48,14 @@ /** * This action can be used to add the record for the persistent action to the cluster state. */ -public class CreatePersistentTaskAction extends Action { + StartPersistentTaskAction.RequestBuilder> { - public static final CreatePersistentTaskAction INSTANCE = new CreatePersistentTaskAction(); - public static final String NAME = "cluster:admin/persistent/create"; + public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/start"; - private CreatePersistentTaskAction() { + private StartPersistentTaskAction() { super(NAME); } @@ -68,18 +73,19 @@ public static class Request extends MasterNodeRequest { private String taskId; + @Nullable private String action; - private PersistentTaskRequest request; + private PersistentTaskParams params; public Request() { } - public Request(String taskId, String action, PersistentTaskRequest request) { + public Request(String taskId, String action, PersistentTaskParams params) { this.taskId = taskId; this.action = action; - this.request = request; + this.params = params; } @Override @@ -87,7 +93,7 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); taskId = in.readString(); action = in.readString(); - request = in.readNamedWriteable(PersistentTaskRequest.class); + params = in.readOptionalNamedWriteable(PersistentTaskParams.class); } @Override @@ -95,7 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(taskId); out.writeString(action); - out.writeNamedWriteable(request); + out.writeOptionalNamedWriteable(params); } @Override @@ -107,9 +113,6 @@ public ActionRequestValidationException validate() { if (this.action == null) { validationException = addValidationError("action must be specified", validationException); } - if (this.request == null) { - validationException = addValidationError("request must be specified", validationException); - } return validationException; } @@ -119,12 +122,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Request request1 = (Request) o; return Objects.equals(taskId, request1.taskId) && Objects.equals(action, request1.action) && - Objects.equals(request, request1.request); + Objects.equals(params, request1.params); } @Override public int hashCode() { - return Objects.hash(taskId, action, request); + return Objects.hash(taskId, action, params); } public String getAction() { @@ -143,20 +146,21 @@ public void setTaskId(String taskId) { this.taskId = taskId; } - public PersistentTaskRequest getRequest() { - return request; + public PersistentTaskParams getParams() { + return params; } - public void setRequest(PersistentTaskRequest request) { - this.request = request; + @Nullable + public void setParams(PersistentTaskParams params) { + this.params = params; } } - public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { - protected RequestBuilder(ElasticsearchClient client, CreatePersistentTaskAction action) { + protected RequestBuilder(ElasticsearchClient client, StartPersistentTaskAction action) { super(client, action, new Request()); } @@ -170,8 +174,8 @@ public RequestBuilder setAction(String action) { return this; } - public RequestBuilder setRequest(PersistentTaskRequest persistentTaskRequest) { - request.setRequest(persistentTaskRequest); + public RequestBuilder setRequest(PersistentTaskParams params) { + request.setParams(params); return this; } @@ -188,7 +192,7 @@ public TransportAction(Settings settings, TransportService transportService, Clu PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, PersistentTasksService persistentTasksService, IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, CreatePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); this.persistentTasksClusterService = persistentTasksClusterService; NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool); @@ -215,7 +219,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.createPersistentTask(request.taskId, request.action, request.request, + persistentTasksClusterService.createPersistentTask(request.taskId, request.action, request.params, new ActionListener>() { @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index 77d8a0a8c2103..6ced2bddf283e 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.test.VersionUtils; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import java.util.ArrayList; import java.util.Arrays; @@ -68,9 +68,9 @@ public void testReassignmentRequired() { assertThat(dumpEvent(event), PersistentTasksClusterService.reassignmentRequired(event, new PersistentTasksClusterService.ExecutorNodeDecider() { @Override - public Assignment getAssignment( - String action, ClusterState currentState, Request request) { - if ("never_assign".equals(((TestRequest) request).getTestParam())) { + public Assignment getAssignment( + String action, ClusterState currentState, Params params) { + if ("never_assign".equals(((TestParams) params).getTestParam())) { return NO_NODE_FOUND; } return randomNodeAssignment(currentState.nodes()); @@ -185,10 +185,10 @@ private ClusterState reassign(ClusterState clusterState) { return PersistentTasksClusterService.reassignTasks(clusterState, logger, new PersistentTasksClusterService.ExecutorNodeDecider() { @Override - public Assignment getAssignment( - String action, ClusterState currentState, Request request) { - TestRequest testRequest = (TestRequest) request; - switch (testRequest.getTestParam()) { + public Assignment getAssignment( + String action, ClusterState currentState, Params params) { + TestParams testParams = (TestParams) params; + switch (testParams.getTestParam()) { case "assign_me": return randomNodeAssignment(currentState.nodes()); case "dont_assign_me": @@ -199,7 +199,7 @@ public Assignment getAssignment( case "assign_one": return assignOnlyOneTaskAtATime(currentState); default: - fail("unknown param " + testRequest.getTestParam()); + fail("unknown param " + testParams.getTestParam()); } return NO_NODE_FOUND; } @@ -293,7 +293,7 @@ private PersistentTasksCustomMetaData removeTasksWithChangingAssignment(Persiste PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); for (PersistentTask task : tasks.tasks()) { // Remove all unassigned tasks that cause changing assignments they might trigger a significant change - if ("never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) && + if ("never_assign".equals(((TestParams) task.getParams()).getTestParam()) && "change me".equals(task.getAssignment().getExplanation())) { logger.info("removed task with changing assignment {}", task.getId()); tasksBuilder.removeTask(task.getId()); @@ -360,7 +360,7 @@ private ClusterState insignificantChange(ClusterState clusterState) { logger.info("removed all unassigned tasks and changed routing table"); if (tasks != null) { for (PersistentTask task : tasks.tasks()) { - if (task.getExecutorNode() == null || "never_assign".equals(((TestRequest) task.getRequest()).getTestParam())) { + if (task.getExecutorNode() == null || "never_assign".equals(((TestParams) task.getParams()).getTestParam())) { tasksBuilder.removeTask(task.getId()); } } @@ -382,7 +382,7 @@ private boolean hasAssignableTasks(PersistentTasksCustomMetaData tasks, Discover } return tasks.tasks().stream().anyMatch(task -> { if (task.getExecutorNode() == null || discoveryNodes.nodeExists(task.getExecutorNode())) { - return "never_assign".equals(((TestRequest) task.getRequest()).getTestParam()) == false; + return "never_assign".equals(((TestParams) task.getParams()).getTestParam()) == false; } return false; }); @@ -404,11 +404,11 @@ private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuil MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, Assignment assignment, String param) { return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, - tasks.addTask(UUIDs.base64UUID(), randomAlphaOfLength(10), new TestRequest(param), assignment).build())); + tasks.addTask(UUIDs.base64UUID(), randomAlphaOfLength(10), new TestParams(param), assignment).build())); } private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node) { - tasks.addTask(UUIDs.base64UUID(), action, new TestRequest(param), new Assignment(node, "explanation: " + action)); + tasks.addTask(UUIDs.base64UUID(), action, new TestParams(param), new Assignment(node, "explanation: " + action)); } private DiscoveryNode newNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index e66b0ca1af6fc..3a4d1bc861cca 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -41,7 +41,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import java.io.IOException; import java.util.ArrayList; @@ -60,7 +60,7 @@ protected PersistentTasksCustomMetaData createTestInstance() { PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); for (int i = 0; i < numberOfTasks; i++) { String taskId = UUIDs.base64UUID(); - tasks.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), + tasks.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams(randomAlphaOfLength(10)), randomAssignment()); if (randomBoolean()) { // From time to time update status @@ -80,7 +80,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Arrays.asList( new Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new), new Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom), - new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new), + new Entry(PersistentTaskParams.class, TestPersistentTasksExecutor.NAME, TestParams::new), new Entry(Task.Status.class, Status.NAME, Status::new) )); } @@ -151,7 +151,7 @@ protected XContentBuilder toXContent(Custom instance, XContentType contentType, private String addRandomTask(Builder builder) { String taskId = UUIDs.base64UUID(); - builder.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest(randomAlphaOfLength(10)), randomAssignment()); + builder.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams(randomAlphaOfLength(10)), randomAssignment()); return taskId; } @@ -162,8 +162,8 @@ private String pickRandomTask(PersistentTasksCustomMetaData.Builder testInstance @Override protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(Arrays.asList( - new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME), - TestRequest::fromXContent), + new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(TestPersistentTasksExecutor.NAME), + TestParams::fromXContent), new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) )); } @@ -188,14 +188,14 @@ public void testSerializationContext() throws Exception { assertEquals(testInstance.tasks().size(), newInstance.tasks().size()); for (PersistentTask testTask : testInstance.tasks()) { - PersistentTask newTask = (PersistentTask) newInstance.getTask(testTask.getId()); + PersistentTask newTask = (PersistentTask) newInstance.getTask(testTask.getId()); assertNotNull(newTask); // Things that should be serialized assertEquals(testTask.getTaskName(), newTask.getTaskName()); assertEquals(testTask.getId(), newTask.getId()); assertEquals(testTask.getStatus(), newTask.getStatus()); - assertEquals(testTask.getRequest(), newTask.getRequest()); + assertEquals(testTask.getParams(), newTask.getParams()); // Things that shouldn't be serialized assertEquals(0, newTask.getAllocationId()); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index b68464636f541..0dddaaa783906 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import java.util.ArrayList; import java.util.Collection; @@ -59,13 +59,14 @@ public void testFullClusterRestart() throws Exception { PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class); int numberOfTasks = randomIntBetween(1, 10); String[] taskIds = new String[numberOfTasks]; - List>> futures = new ArrayList<>(numberOfTasks); + List>> futures = new ArrayList<>(numberOfTasks); for (int i = 0; i < numberOfTasks; i++) { - PlainActionFuture> future = new PlainActionFuture<>(); + PlainActionFuture> future = new PlainActionFuture<>(); futures.add(future); taskIds[i] = UUIDs.base64UUID(); - service.startPersistentTask(taskIds[i], TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + service.startPersistentTask(taskIds[i], TestPersistentTasksExecutor.NAME, randomBoolean() ? null : new TestParams("Blah"), + future); } for (int i = 0; i < numberOfTasks; i++) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 40cecfb293627..604581e1e56f5 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -33,7 +33,7 @@ import org.elasticsearch.persistent.PersistentTasksService.WaitForPersistentTaskStatusListener; import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder; import org.junit.After; @@ -69,15 +69,15 @@ public void cleanup() throws Exception { assertNoRunningTasks(); } - public static class WaitForPersistentTaskStatusFuture - extends PlainActionFuture> - implements WaitForPersistentTaskStatusListener { + public static class WaitForPersistentTaskStatusFuture + extends PlainActionFuture> + implements WaitForPersistentTaskStatusListener { } public void testPersistentActionFailure() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PlainActionFuture> future = new PlainActionFuture<>(); - persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + PlainActionFuture> future = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); long allocationId = future.get().getAllocationId(); assertBusy(() -> { // Wait for the task to start @@ -106,29 +106,31 @@ public void testPersistentActionFailure() throws Exception { public void testPersistentActionCompletion() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PlainActionFuture> future = new PlainActionFuture<>(); - persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); - long taskId = future.get().getAllocationId(); + PlainActionFuture> future = new PlainActionFuture<>(); + String taskId = UUIDs.base64UUID(); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); + long allocationId = future.get().getAllocationId(); assertBusy(() -> { // Wait for the task to start assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() .getTasks().size(), equalTo(1)); }); TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") - .get().getTasks().get(0); + .setDetailed(true).get().getTasks().get(0); logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); - // Verifying parent - assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(taskId)); + // Verifying parent and description + assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(allocationId)); assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); + assertThat(firstRunningTask.getDescription(), equalTo("id=" + taskId)); stopOrCancelTask(firstRunningTask.getTaskId()); } public void testPersistentActionWithNoAvailableNode() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PlainActionFuture> future = new PlainActionFuture<>(); - TestRequest testRequest = new TestRequest("Blah"); - testRequest.setExecutorNodeAttr("test"); - persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, testRequest, future); + PlainActionFuture> future = new PlainActionFuture<>(); + TestParams testParams = new TestParams("Blah"); + testParams.setExecutorNodeAttr("test"); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, testParams, future); String taskId = future.get().getId(); Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); @@ -161,8 +163,8 @@ public void testPersistentActionWithNoAvailableNode() throws Exception { public void testPersistentActionStatusUpdate() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PlainActionFuture> future = new PlainActionFuture<>(); - persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + PlainActionFuture> future = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); String taskId = future.get().getId(); assertBusy(() -> { @@ -219,13 +221,13 @@ public void testPersistentActionStatusUpdate() throws Exception { public void testCreatePersistentTaskWithDuplicateId() throws Exception { PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); - PlainActionFuture> future = new PlainActionFuture<>(); + PlainActionFuture> future = new PlainActionFuture<>(); String taskId = UUIDs.base64UUID(); - persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); future.get(); - PlainActionFuture> future2 = new PlainActionFuture<>(); - persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestRequest("Blah"), future2); + PlainActionFuture> future2 = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future2); assertThrows(future2, ResourceAlreadyExistsException.class); assertBusy(() -> { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java index eb18e4fdf8652..47bec50e811c5 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java @@ -33,8 +33,8 @@ public class PersistentTasksExecutorResponseTests extends AbstractStreamableTest protected PersistentTaskResponse createTestInstance() { if (randomBoolean()) { return new PersistentTaskResponse( - new PersistentTask(UUIDs.base64UUID(), randomAsciiOfLength(10), - new TestPersistentTasksPlugin.TestRequest("test"), + new PersistentTask(UUIDs.base64UUID(), randomAsciiOfLength(10), + new TestPersistentTasksPlugin.TestParams("test"), randomLong(), PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT)); } else { return new PersistentTaskResponse(null); @@ -49,7 +49,8 @@ protected PersistentTaskResponse createBlankInstance() { @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Collections.singletonList( - new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, TestPersistentTasksPlugin.TestPersistentTasksExecutor.NAME, TestPersistentTasksPlugin.TestRequest::new) + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, + TestPersistentTasksPlugin.TestPersistentTasksExecutor.NAME, TestPersistentTasksPlugin.TestParams::new) )); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 8d1960bb8054d..13bf1e5c2ed1f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -32,12 +32,13 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import java.io.IOException; import java.util.ArrayList; @@ -48,6 +49,10 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -73,12 +78,18 @@ private DiscoveryNodes createTestNodes(int nonLocalNodesCount, Settings settings public void testStartTask() throws Exception { ClusterService clusterService = createClusterService(); PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); - @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); when(action.getTaskName()).thenReturn("test"); + int nonLocalNodesCount = randomInt(10); + // need to account for 5 original tasks on each node and their relocations + for (int i = 0; i < (nonLocalNodesCount + 1) * 10; i++) { + TaskId parentId = new TaskId("cluster", i); + when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any())).thenReturn( + new TestPersistentTasksPlugin.TestTask(i, "persistent", "test", "", parentId)); + } PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); - int nonLocalNodesCount = randomInt(10); MockExecutor executor = new MockExecutor(); PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, registry, new TaskManager(Settings.EMPTY), executor); @@ -90,11 +101,11 @@ public void testStartTask() throws Exception { boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { - tasks.addTask(UUIDs.base64UUID(), "test_action", new TestRequest("other_" + i), + tasks.addTask(UUIDs.base64UUID(), "test_action", new TestParams("other_" + i), new Assignment("other_node_" + randomInt(nonLocalNodesCount), "test assignment on other node")); if (added == false && randomBoolean()) { added = true; - tasks.addTask(UUIDs.base64UUID(), "test", new TestRequest("this_param"), + tasks.addTask(UUIDs.base64UUID(), "test", new TestParams("this_param"), new Assignment("this_node", "test assignment on this node")); } } @@ -115,7 +126,7 @@ public void testStartTask() throws Exception { // Add task on some other node state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "some_other_node"); + newClusterState = addTask(state, "test", null, "some_other_node"); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); // Make sure action wasn't called again @@ -123,7 +134,7 @@ public void testStartTask() throws Exception { // Start another task on this node state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest("this_param"), "this_node"); + newClusterState = addTask(state, "test", new TestParams("this_param"), "this_node"); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); // Make sure action was called this time @@ -138,7 +149,7 @@ public void testStartTask() throws Exception { // Add task on some other node state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "some_other_node"); + newClusterState = addTask(state, "test", null, "some_other_node"); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); // Make sure action wasn't called again @@ -176,9 +187,11 @@ public void sendCompletionNotification(String taskId, Exception failure, ActionL fail("Shouldn't be called during Cluster State cancellation"); } }; - @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); when(action.getTaskName()).thenReturn("test"); + when(action.createTask(anyLong(), anyString(), anyString(), any(), any())) + .thenReturn(new TestPersistentTasksPlugin.TestTask(1, "persistent", "test", "", new TaskId("cluster", 1))); PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); int nonLocalNodesCount = randomInt(10); @@ -193,7 +206,7 @@ public void sendCompletionNotification(String taskId, Exception failure, ActionL ClusterState newClusterState = state; // Allocate first task state = newClusterState; - newClusterState = addTask(state, "test", new TestRequest(), "this_node"); + newClusterState = addTask(state, "test", null, "this_node"); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); // Check the the task is know to the task manager @@ -236,12 +249,12 @@ public void sendCompletionNotification(String taskId, Exception failure, ActionL } - private ClusterState addTask(ClusterState state, String action, Request request, - String node) { + private ClusterState addTask(ClusterState state, String action, Params params, + String node) { PersistentTasksCustomMetaData.Builder builder = PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, - builder.addTask(UUIDs.base64UUID(), action, request, new Assignment(node, "test assignment")).build())).build(); + builder.addTask(UUIDs.base64UUID(), action, params, new Assignment(node, "test assignment")).build())).build(); } private ClusterState reallocateTask(ClusterState state, String taskId, String node) { @@ -261,12 +274,12 @@ private ClusterState removeTask(ClusterState state, String taskId) { } private class Execution { - private final PersistentTaskRequest request; + private final PersistentTaskParams params; private final AllocatedPersistentTask task; private final PersistentTasksExecutor holder; - Execution(PersistentTaskRequest request, AllocatedPersistentTask task, PersistentTasksExecutor holder) { - this.request = request; + Execution(PersistentTaskParams params, AllocatedPersistentTask task, PersistentTasksExecutor holder) { + this.params = params; this.task = task; this.holder = holder; } @@ -280,9 +293,9 @@ private class MockExecutor extends NodePersistentTasksExecutor { } @Override - public void executeTask(Request request, AllocatedPersistentTask task, - PersistentTasksExecutor action) { - executions.add(new Execution(request, task, action)); + public void executeTask(Params params, AllocatedPersistentTask task, + PersistentTasksExecutor executor) { + executions.add(new Execution(params, task, executor)); } public Execution get(int i) { diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java index b4e7fab62ec82..3b0fc2a3d0495 100644 --- a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -21,9 +21,9 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; -import org.elasticsearch.persistent.CreatePersistentTaskAction.Request; +import org.elasticsearch.persistent.StartPersistentTaskAction.Request; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestRequest; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import org.elasticsearch.test.AbstractStreamableTestCase; import java.util.Collections; @@ -32,17 +32,19 @@ public class StartPersistentActionRequestTests extends AbstractStreamableTestCas @Override protected Request createTestInstance() { - TestRequest testRequest = new TestRequest(); + TestParams testParams; if (randomBoolean()) { - testRequest.setTestParam(randomAlphaOfLengthBetween(1, 20)); + testParams = new TestParams(); + if (randomBoolean()) { + testParams.setTestParam(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + testParams.setExecutorNodeAttr(randomAlphaOfLengthBetween(1, 20)); + } + } else { + testParams = null; } - if (randomBoolean()) { - testRequest.setParentTask(randomAlphaOfLengthBetween(1, 20), randomLong()); - } - if (randomBoolean()) { - testRequest.setExecutorNodeAttr(randomAlphaOfLengthBetween(1, 20)); - } - return new Request(UUIDs.base64UUID(), randomAlphaOfLengthBetween(1, 20), new TestRequest()); + return new Request(UUIDs.base64UUID(), randomAlphaOfLengthBetween(1, 20), testParams); } @Override @@ -53,7 +55,7 @@ protected Request createBlankInstance() { @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Collections.singletonList( - new Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new) + new Entry(PersistentTaskParams.class, TestPersistentTasksExecutor.NAME, TestParams::new) )); } } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index ae7b447a26d93..d2c4dae371783 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.TaskOperationFailure; @@ -92,7 +91,7 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { public List> getActions() { return Arrays.asList( new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), - new ActionHandler<>(CreatePersistentTaskAction.INSTANCE, CreatePersistentTaskAction.TransportAction.class), + new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), new ActionHandler<>(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class), new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), new ActionHandler<>(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class) @@ -118,7 +117,7 @@ public Collection createComponents(Client client, ClusterService cluster @Override public List getNamedWriteables() { return Arrays.asList( - new NamedWriteableRegistry.Entry(PersistentTaskRequest.class, TestPersistentTasksExecutor.NAME, TestRequest::new), + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TestPersistentTasksExecutor.NAME, TestParams::new), new NamedWriteableRegistry.Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new), new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, @@ -134,16 +133,16 @@ public List getNamedXContent() { return Arrays.asList( new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE), PersistentTasksCustomMetaData::fromXContent), - new NamedXContentRegistry.Entry(PersistentTaskRequest.class, new ParseField(TestPersistentTasksExecutor.NAME), - TestRequest::fromXContent), + new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(TestPersistentTasksExecutor.NAME), + TestParams::fromXContent), new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) ); } - public static class TestRequest extends PersistentTaskRequest { + public static class TestParams implements PersistentTaskParams { - public static final ConstructingObjectParser REQUEST_PARSER = - new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new TestRequest((String) args[0])); + public static final ConstructingObjectParser REQUEST_PARSER = + new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new TestParams((String) args[0])); static { REQUEST_PARSER.declareString(constructorArg(), new ParseField("param")); @@ -155,21 +154,18 @@ public static class TestRequest extends PersistentTaskRequest { private String testParam = null; - public TestRequest() { + public TestParams() { } - public TestRequest(String testParam) { + public TestParams(String testParam) { this.testParam = testParam; } - public TestRequest(StreamInput in) throws IOException { - readFrom(in); - } - - @Override - public ActionRequestValidationException validate() { - return null; + public TestParams(StreamInput in) throws IOException { + executorNodeAttr = in.readOptionalString(); + responseNode = in.readOptionalString(); + testParam = in.readOptionalString(); } @Override @@ -195,20 +191,11 @@ public String getTestParam() { @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); out.writeOptionalString(executorNodeAttr); out.writeOptionalString(responseNode); out.writeOptionalString(testParam); } - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - executorNodeAttr = in.readOptionalString(); - responseNode = in.readOptionalString(); - testParam = in.readOptionalString(); - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -217,7 +204,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public static TestRequest fromXContent(XContentParser parser) throws IOException { + public static TestParams fromXContent(XContentParser parser) throws IOException { return REQUEST_PARSER.parse(parser, null); } @@ -225,7 +212,7 @@ public static TestRequest fromXContent(XContentParser parser) throws IOException public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - TestRequest that = (TestRequest) o; + TestParams that = (TestParams) o; return Objects.equals(executorNodeAttr, that.executorNodeAttr) && Objects.equals(responseNode, that.responseNode) && Objects.equals(testParam, that.testParam); @@ -235,11 +222,6 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(executorNodeAttr, responseNode, testParam); } - - @Override - public Task createTask(long id, String type, String action, TaskId parentTaskId) { - return new TestTask(id, type, action, getDescription(), parentTaskId); - } } public static class Status implements Task.Status { @@ -312,7 +294,7 @@ public int hashCode() { } - public static class TestPersistentTasksExecutor extends PersistentTasksExecutor { + public static class TestPersistentTasksExecutor extends PersistentTasksExecutor { public static final String NAME = "cluster:admin/persistent/test"; private final ClusterService clusterService; @@ -323,12 +305,12 @@ public TestPersistentTasksExecutor(Settings settings, ClusterService clusterServ } @Override - public Assignment getAssignment(TestRequest request, ClusterState clusterState) { - if (request.getExecutorNodeAttr() == null) { - return super.getAssignment(request, clusterState); + public Assignment getAssignment(TestParams params, ClusterState clusterState) { + if (params == null || params.getExecutorNodeAttr() == null) { + return super.getAssignment(params, clusterState); } else { DiscoveryNode executorNode = selectLeastLoadedNode(clusterState, - discoveryNode -> request.getExecutorNodeAttr().equals(discoveryNode.getAttributes().get("test_attr"))); + discoveryNode -> params.getExecutorNodeAttr().equals(discoveryNode.getAttributes().get("test_attr"))); if (executorNode != null) { return new Assignment(executorNode.getId(), "test assignment"); } else { @@ -339,7 +321,7 @@ public Assignment getAssignment(TestRequest request, ClusterState clusterState) } @Override - protected void nodeOperation(AllocatedPersistentTask task, TestRequest request) { + protected void nodeOperation(AllocatedPersistentTask task, TestParams params) { logger.info("started node operation for the task {}", task); try { TestTask testTask = (TestTask) task; @@ -399,6 +381,12 @@ public void onFailure(Exception e) { task.markAsFailed(e); } } + + @Override + protected AllocatedPersistentTask createTask(long id, String type, String action, TaskId parentTaskId, + PersistentTask task) { + return new TestTask(id, type, action, getDescription(task), parentTaskId); + } } public static class TestTaskAction extends Action { From 4771965931500932a2cb6973fb64ea874d6f4fb9 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 12 Apr 2017 16:02:48 +0200 Subject: [PATCH 29/50] Use task builder instead of creating persistent tasks directly. --- .../persistent/PersistentTasksCustomMetaData.java | 9 ++++++--- .../persistent/PersistentTasksCustomMetaDataTests.java | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 18d8b00fea5a6..ab7357940dd97 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -499,7 +499,6 @@ public long getLastAllocationId() { return lastAllocationId; } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("last_allocation_id", lastAllocationId); @@ -524,10 +523,10 @@ public static class Builder { private long lastAllocationId; private boolean changed; - public Builder() { + private Builder() { } - public Builder(PersistentTasksCustomMetaData tasksInProgress) { + private Builder(PersistentTasksCustomMetaData tasksInProgress) { if (tasksInProgress != null) { tasks.putAll(tasksInProgress.tasks); lastAllocationId = tasksInProgress.lastAllocationId; @@ -536,6 +535,10 @@ public Builder(PersistentTasksCustomMetaData tasksInProgress) { } } + public long getLastAllocationId() { + return lastAllocationId; + } + private Builder setLastAllocationId(long currentId) { this.lastAllocationId = currentId; return this; diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index 3a4d1bc861cca..e36fc4c379c49 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -87,7 +87,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { @Override protected Custom makeTestChanges(Custom testInstance) { - Builder builder = new Builder((PersistentTasksCustomMetaData) testInstance); + Builder builder = PersistentTasksCustomMetaData.builder((PersistentTasksCustomMetaData) testInstance); switch (randomInt(3)) { case 0: addRandomTask(builder); @@ -209,9 +209,9 @@ public void testBuilder() { for (int i = 0; i < randomIntBetween(10, 100); i++) { final Builder builder; if (randomBoolean()) { - builder = new Builder(); + builder = PersistentTasksCustomMetaData.builder(); } else { - builder = new Builder(persistentTasks); + builder = PersistentTasksCustomMetaData.builder(persistentTasks); } boolean changed = false; for (int j = 0; j < randomIntBetween(1, 10); j++) { From fc524bc9b58d96a462c977d14325a021ab29c338 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 13 Apr 2017 09:49:55 -0400 Subject: [PATCH 30/50] Persistent Tasks: force writeable name of params and status to be the same as their task (#1072) Changes persistent task serialization and forces params and status to have the same writeable name as the task itself. --- .../PersistentTasksCustomMetaData.java | 90 +++++++++++++------ .../persistent/StartPersistentTaskAction.java | 34 ++++--- .../UpdatePersistentTaskStatusAction.java | 17 +++- .../PersistentTasksClusterServiceTests.java | 26 +++--- .../PersistentTasksCustomMetaDataTests.java | 4 +- .../persistent/PersistentTasksExecutorIT.java | 4 +- .../PersistentTasksExecutorResponseTests.java | 7 +- .../PersistentTasksNodeServiceTests.java | 13 +-- .../persistent/TestPersistentTasksPlugin.java | 7 +- .../UpdatePersistentTaskRequestTests.java | 3 +- 10 files changed, 130 insertions(+), 75 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index ab7357940dd97..7dfccfdddd309 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -82,16 +82,21 @@ public PersistentTasksCustomMetaData(long lastAllocationId, Map ASSIGNMENT_PARSER = new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); - private static final NamedObjectParser PARAMS_PARSER = - (XContentParser p, Void c, String name) -> p.namedObject(PersistentTaskParams.class, name, null); - private static final NamedObjectParser STATUS_PARSER = - (XContentParser p, Void c, String name) -> p.namedObject(Status.class, name, null); + private static final NamedObjectParser, Void> TASK_DESCRIPTION_PARSER; static { // Tasks parser initialization PERSISTENT_TASKS_PARSER.declareLong(Builder::setLastAllocationId, new ParseField("last_allocation_id")); PERSISTENT_TASKS_PARSER.declareObjectArray(Builder::setTasks, PERSISTENT_TASK_PARSER, new ParseField("tasks")); + // Task description parser initialization + ObjectParser, String> parser = new ObjectParser<>("named"); + parser.declareObject(TaskDescriptionBuilder::setParams, + (p, c) -> p.namedObject(PersistentTaskParams.class, c, null), new ParseField("params")); + parser.declareObject(TaskDescriptionBuilder::setStatus, + (p, c) -> p.namedObject(Status.class, c, null), new ParseField("status")); + TASK_DESCRIPTION_PARSER = (XContentParser p, Void c, String name) -> parser.parse(p, new TaskDescriptionBuilder<>(name), name); + // Assignment parser ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("executor_node")); ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("explanation")); @@ -100,28 +105,46 @@ public PersistentTasksCustomMetaData(long lastAllocationId, Map taskBuilder, List objects) -> { - if (objects.size() != 1) { - throw new IllegalArgumentException("only one params per task is allowed"); - } - taskBuilder.setParams(objects.get(0)); - }, PARAMS_PARSER, new ParseField("params")); PERSISTENT_TASK_PARSER.declareNamedObjects( - (TaskBuilder taskBuilder, List objects) -> { + (TaskBuilder taskBuilder, List> objects) -> { if (objects.size() != 1) { - throw new IllegalArgumentException("only one status per task is allowed"); + throw new IllegalArgumentException("only one task description per task is allowed"); } - taskBuilder.setStatus(objects.get(0)); - }, STATUS_PARSER, new ParseField("status")); - - + TaskDescriptionBuilder builder = objects.get(0); + taskBuilder.setTaskName(builder.taskName); + taskBuilder.setParams(builder.params); + taskBuilder.setStatus(builder.status); + }, TASK_DESCRIPTION_PARSER, new ParseField("task")); PERSISTENT_TASK_PARSER.declareObject(TaskBuilder::setAssignment, ASSIGNMENT_PARSER, new ParseField("assignment")); PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationIdOnLastStatusUpdate, new ParseField("allocation_id_on_last_status_update")); } + /** + * Private builder used in XContent parser to build task-specific portion (params and status) + */ + private static class TaskDescriptionBuilder { + private final String taskName; + private Params params; + private Status status; + + private TaskDescriptionBuilder(String taskName) { + this.taskName = taskName; + } + + private TaskDescriptionBuilder setParams(Params params) { + this.params = params; + return this; + } + + private TaskDescriptionBuilder setStatus(Status status) { + this.status = status; + return this; + } + } + + public Collection> tasks() { return this.tasks.values(); } @@ -279,6 +302,18 @@ private PersistentTask(String id, long allocationId, String taskName, Params par this.status = status; this.assignment = assignment; this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; + if (params != null) { + if (params.getWriteableName().equals(taskName) == false) { + throw new IllegalArgumentException("params have to have the same writeable name as task. params: " + + params.getWriteableName() + " task: " + taskName); + } + } + if (status != null) { + if (status.getWriteableName().equals(taskName) == false) { + throw new IllegalArgumentException("status has to have the same writeable name as task. status: " + + status.getWriteableName() + " task: " + taskName); + } + } } @SuppressWarnings("unchecked") @@ -384,21 +419,20 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params xPa builder.startObject(); { builder.field("id", id); - builder.field("name", taskName); - if (params != null) { - builder.startObject("params"); - { - builder.field(params.getWriteableName(), params, xParams); - } - builder.endObject(); - } - if (status != null) { - builder.startObject("status"); + builder.startObject("task"); + { + builder.startObject(taskName); { - builder.field(status.getWriteableName(), status, xParams); + if (params != null) { + builder.field("params", params, xParams); + } + if (status != null) { + builder.field("status", status, xParams); + } } builder.endObject(); } + builder.endObject(); if (API_CONTEXT.equals(xParams.param(MetaData.CONTEXT_MODE_PARAM, API_CONTEXT))) { // These are transient values that shouldn't be persisted to gateway cluster state or snapshot diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java index 40d36b9a439d6..3b988939879c5 100644 --- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -74,7 +74,7 @@ public static class Request extends MasterNodeRequest { private String taskId; @Nullable - private String action; + private String taskName; private PersistentTaskParams params; @@ -82,9 +82,9 @@ public Request() { } - public Request(String taskId, String action, PersistentTaskParams params) { + public Request(String taskId, String taskName, PersistentTaskParams params) { this.taskId = taskId; - this.action = action; + this.taskName = taskName; this.params = params; } @@ -92,7 +92,7 @@ public Request(String taskId, String action, PersistentTaskParams params) { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); taskId = in.readString(); - action = in.readString(); + taskName = in.readString(); params = in.readOptionalNamedWriteable(PersistentTaskParams.class); } @@ -100,7 +100,7 @@ public void readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(taskId); - out.writeString(action); + out.writeString(taskName); out.writeOptionalNamedWriteable(params); } @@ -110,9 +110,15 @@ public ActionRequestValidationException validate() { if (this.taskId == null) { validationException = addValidationError("task id must be specified", validationException); } - if (this.action == null) { + if (this.taskName == null) { validationException = addValidationError("action must be specified", validationException); } + if (params != null) { + if (params.getWriteableName().equals(taskName) == false) { + validationException = addValidationError("params have to have the same writeable name as task. params: " + + params.getWriteableName() + " task: " + taskName, validationException); + } + } return validationException; } @@ -121,21 +127,21 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request1 = (Request) o; - return Objects.equals(taskId, request1.taskId) && Objects.equals(action, request1.action) && + return Objects.equals(taskId, request1.taskId) && Objects.equals(taskName, request1.taskName) && Objects.equals(params, request1.params); } @Override public int hashCode() { - return Objects.hash(taskId, action, params); + return Objects.hash(taskId, taskName, params); } - public String getAction() { - return action; + public String getTaskName() { + return taskName; } - public void setAction(String action) { - this.action = action; + public void setTaskName(String taskName) { + this.taskName = taskName; } public String getTaskId() { @@ -170,7 +176,7 @@ public RequestBuilder setTaskId(String taskId) { } public RequestBuilder setAction(String action) { - request.setAction(action); + request.setTaskName(action); return this; } @@ -219,7 +225,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.createPersistentTask(request.taskId, request.action, request.params, + persistentTasksClusterService.createPersistentTask(request.taskId, request.taskName, request.params, new ActionListener>() { @Override diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java index 8c1de889e0d48..53bc9afd0fdf5 100644 --- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -38,13 +37,14 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import java.io.IOException; import java.util.Objects; +import static org.elasticsearch.action.ValidateActions.addValidationError; + public class UpdatePersistentTaskStatusAction extends Action { @@ -69,7 +69,7 @@ public PersistentTaskResponse newResponse() { public static class Request extends MasterNodeRequest { private String taskId; - private long allocationId; + private long allocationId = -1L; private Task.Status status; public Request() { @@ -112,7 +112,16 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { - return null; + ActionRequestValidationException validationException = null; + if (this.taskId == null) { + validationException = addValidationError("task id must be specified", validationException); + } + if (this.allocationId == -1L) { + validationException = addValidationError("allocationId must be specified", validationException); + } + // We cannot really check if status has the same type as task because we don't have access + // to the task here. We will check it when we try to update the task + return validationException; } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index 6ced2bddf283e..1169ff91e1308 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import java.util.ArrayList; import java.util.Arrays; @@ -93,7 +94,7 @@ public void testReassignConsidersClusterStateUpdates() { addTestNodes(nodes, randomIntBetween(1, 10)); int numberOfTasks = randomIntBetween(2, 40); for (int i = 0; i < numberOfTasks; i++) { - addTask(tasks, "should_assign", "assign_one", randomBoolean() ? null : "no_longer_exits"); + addTask(tasks, "assign_one", randomBoolean() ? null : "no_longer_exits"); } MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); @@ -117,14 +118,14 @@ public void testReassignTasks() { switch (randomInt(2)) { case 0: // add an unassigned task that should get assigned because it's assigned to a non-existing node or unassigned - addTask(tasks, "should_assign", "assign_me", randomBoolean() ? null : "no_longer_exits"); + addTask(tasks, "assign_me", randomBoolean() ? null : "no_longer_exits"); break; case 1: // add a task assigned to non-existing node that should not get assigned - addTask(tasks, "should_not_assign", "dont_assign_me", randomBoolean() ? null : "no_longer_exits"); + addTask(tasks, "dont_assign_me", randomBoolean() ? null : "no_longer_exits"); break; case 2: - addTask(tasks, "assign_one", "assign_one", randomBoolean() ? null : "no_longer_exits"); + addTask(tasks, "assign_one", randomBoolean() ? null : "no_longer_exits"); break; } @@ -143,8 +144,8 @@ public void testReassignTasks() { for (PersistentTask task : tasksInProgress.tasks()) { // explanation should correspond to the action name - switch (task.getTaskName()) { - case "should_assign": + switch (((TestParams) task.getParams()).getTestParam()) { + case "assign_me": assertThat(task.getExecutorNode(), notNullValue()); assertThat(task.isAssigned(), equalTo(true)); if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { @@ -154,7 +155,7 @@ public void testReassignTasks() { clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); break; - case "should_not_assign": + case "dont_assign_me": assertThat(task.getExecutorNode(), nullValue()); assertThat(task.isAssigned(), equalTo(false)); assertThat(task.getAssignment().getExplanation(), equalTo("no appropriate nodes found for the assignment")); @@ -210,7 +211,9 @@ public Assignment getAssignment( private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { DiscoveryNodes nodes = clusterState.nodes(); PersistentTasksCustomMetaData tasksInProgress = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); - if (tasksInProgress.findTasks("assign_one", task -> nodes.nodeExists(task.getExecutorNode())).isEmpty()) { + if (tasksInProgress.findTasks(TestPersistentTasksExecutor.NAME, task -> + "assign_one".equals(((TestParams) task.getParams()).getTestParam()) && + nodes.nodeExists(task.getExecutorNode())).isEmpty()) { return randomNodeAssignment(clusterState.nodes()); } else { return new Assignment(null, "only one task can be assigned at a time"); @@ -404,11 +407,12 @@ private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuil MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, Assignment assignment, String param) { return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, - tasks.addTask(UUIDs.base64UUID(), randomAlphaOfLength(10), new TestParams(param), assignment).build())); + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams(param), assignment).build())); } - private void addTask(PersistentTasksCustomMetaData.Builder tasks, String action, String param, String node) { - tasks.addTask(UUIDs.base64UUID(), action, new TestParams(param), new Assignment(node, "explanation: " + action)); + private void addTask(PersistentTasksCustomMetaData.Builder tasks, String param, String node) { + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams(param), + new Assignment(node, "explanation: " + param)); } private DiscoveryNode newNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index e36fc4c379c49..ea33d6cfe9f5e 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -81,7 +81,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() { new Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new), new Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom), new Entry(PersistentTaskParams.class, TestPersistentTasksExecutor.NAME, TestParams::new), - new Entry(Task.Status.class, Status.NAME, Status::new) + new Entry(Task.Status.class, TestPersistentTasksExecutor.NAME, Status::new) )); } @@ -164,7 +164,7 @@ protected NamedXContentRegistry xContentRegistry() { return new NamedXContentRegistry(Arrays.asList( new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(TestPersistentTasksExecutor.NAME), TestParams::fromXContent), - new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) + new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(TestPersistentTasksExecutor.NAME), Status::fromXContent) )); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 604581e1e56f5..7790de710bf88 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -203,9 +203,9 @@ public void testPersistentActionStatusUpdate() throws Exception { assertThrows(future1, IllegalStateException.class, "timed out after 10ms"); PlainActionFuture> failedUpdateFuture = new PlainActionFuture<>(); - persistentTasksService.updateStatus(taskId, -1, new Status("should fail"), failedUpdateFuture); + persistentTasksService.updateStatus(taskId, -2, new Status("should fail"), failedUpdateFuture); assertThrows(failedUpdateFuture, ResourceNotFoundException.class, "the task with id " + taskId + - " and allocation id -1 doesn't exist"); + " and allocation id -2 doesn't exist"); // Wait for the task to disappear WaitForPersistentTaskStatusFuture future2 = new WaitForPersistentTaskStatusFuture<>(); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java index 47bec50e811c5..ada0e24baa7bd 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.test.AbstractStreamableTestCase; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import java.util.Collections; @@ -33,7 +34,7 @@ public class PersistentTasksExecutorResponseTests extends AbstractStreamableTest protected PersistentTaskResponse createTestInstance() { if (randomBoolean()) { return new PersistentTaskResponse( - new PersistentTask(UUIDs.base64UUID(), randomAsciiOfLength(10), + new PersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestPersistentTasksPlugin.TestParams("test"), randomLong(), PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT)); } else { @@ -50,7 +51,7 @@ protected PersistentTaskResponse createBlankInstance() { protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Collections.singletonList( new NamedWriteableRegistry.Entry(PersistentTaskParams.class, - TestPersistentTasksPlugin.TestPersistentTasksExecutor.NAME, TestPersistentTasksPlugin.TestParams::new) + TestPersistentTasksExecutor.NAME, TestPersistentTasksPlugin.TestParams::new) )); } -} +} \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 13bf1e5c2ed1f..95b0ec87669f1 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import java.io.IOException; import java.util.ArrayList; @@ -80,7 +81,7 @@ public void testStartTask() throws Exception { PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); - when(action.getTaskName()).thenReturn("test"); + when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); int nonLocalNodesCount = randomInt(10); // need to account for 5 original tasks on each node and their relocations for (int i = 0; i < (nonLocalNodesCount + 1) * 10; i++) { @@ -101,11 +102,11 @@ public void testStartTask() throws Exception { boolean added = false; if (nonLocalNodesCount > 0) { for (int i = 0; i < randomInt(5); i++) { - tasks.addTask(UUIDs.base64UUID(), "test_action", new TestParams("other_" + i), + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("other_" + i), new Assignment("other_node_" + randomInt(nonLocalNodesCount), "test assignment on other node")); if (added == false && randomBoolean()) { added = true; - tasks.addTask(UUIDs.base64UUID(), "test", new TestParams("this_param"), + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("this_param"), new Assignment("this_node", "test assignment on this node")); } } @@ -126,7 +127,7 @@ public void testStartTask() throws Exception { // Add task on some other node state = newClusterState; - newClusterState = addTask(state, "test", null, "some_other_node"); + newClusterState = addTask(state, TestPersistentTasksExecutor.NAME, null, "some_other_node"); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); // Make sure action wasn't called again @@ -134,7 +135,7 @@ public void testStartTask() throws Exception { // Start another task on this node state = newClusterState; - newClusterState = addTask(state, "test", new TestParams("this_param"), "this_node"); + newClusterState = addTask(state, TestPersistentTasksExecutor.NAME, new TestParams("this_param"), "this_node"); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); // Make sure action was called this time @@ -149,7 +150,7 @@ public void testStartTask() throws Exception { // Add task on some other node state = newClusterState; - newClusterState = addTask(state, "test", null, "some_other_node"); + newClusterState = addTask(state, TestPersistentTasksExecutor.NAME, null, "some_other_node"); coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); // Make sure action wasn't called again diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index d2c4dae371783..5f3b6c13aaa4b 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -124,7 +124,7 @@ public List getNamedWriteables() { PersistentTasksCustomMetaData::new), new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom), - new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) + new NamedWriteableRegistry.Entry(Task.Status.class, TestPersistentTasksExecutor.NAME, Status::new) ); } @@ -135,7 +135,7 @@ public List getNamedXContent() { PersistentTasksCustomMetaData::fromXContent), new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(TestPersistentTasksExecutor.NAME), TestParams::fromXContent), - new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(Status.NAME), Status::fromXContent) + new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(TestPersistentTasksExecutor.NAME), Status::fromXContent) ); } @@ -225,7 +225,6 @@ public int hashCode() { } public static class Status implements Task.Status { - public static final String NAME = "test"; private final String phase; @@ -246,7 +245,7 @@ public Status(StreamInput in) throws IOException { @Override public String getWriteableName() { - return NAME; + return TestPersistentTasksExecutor.NAME; } @Override diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java index fab4e2e2a758f..6e20bb0009732 100644 --- a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.test.AbstractStreamableTestCase; import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction.Request; import java.util.Collections; @@ -42,7 +43,7 @@ protected Request createBlankInstance() { @Override protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(Collections.singletonList( - new NamedWriteableRegistry.Entry(Task.Status.class, Status.NAME, Status::new) + new NamedWriteableRegistry.Entry(Task.Status.class, TestPersistentTasksExecutor.NAME, Status::new) )); } } From 76cd7b1eb2cf7f87081c24ae6362ae1d9dec6100 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 18 Apr 2017 12:50:05 +0100 Subject: [PATCH 31/50] Fixes compile errors in Eclipse due to generics PersistentTasksCustomMetadata was using a generic param named `Params`. This conflicted with the imported interface `ToXContent.Params`. The java compiler was preferring the generic param over the interface so everything was fine but Eclipse apparently prefers the interface int his case which was screwing up the Hierarchy and causing compile errors in Eclipse. This changes fixes it by renaming the Generic param to `P` --- .../PersistentTasksCustomMetaData.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index 7dfccfdddd309..c24dd6c1b2853 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -266,12 +266,12 @@ public String toString() { /** * A record that represents a single running persistent task */ - public static class PersistentTask implements Writeable, ToXContent { + public static class PersistentTask

implements Writeable, ToXContent { private final String id; private final long allocationId; private final String taskName; @Nullable - private final Params params; + private final P params; @Nullable private final Status status; private final Assignment assignment; @@ -279,21 +279,21 @@ public static class PersistentTask implemen private final Long allocationIdOnLastStatusUpdate; - public PersistentTask(String id, String taskName, Params params, long allocationId, Assignment assignment) { + public PersistentTask(String id, String taskName, P params, long allocationId, Assignment assignment) { this(id, allocationId, taskName, params, null, assignment, null); } - public PersistentTask(PersistentTask task, long allocationId, Assignment assignment) { + public PersistentTask(PersistentTask

task, long allocationId, Assignment assignment) { this(task.id, allocationId, task.taskName, task.params, task.status, assignment, task.allocationId); } - public PersistentTask(PersistentTask task, Status status) { + public PersistentTask(PersistentTask

task, Status status) { this(task.id, task.allocationId, task.taskName, task.params, status, task.assignment, task.allocationId); } - private PersistentTask(String id, long allocationId, String taskName, Params params, + private PersistentTask(String id, long allocationId, String taskName, P params, Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { this.id = id; this.allocationId = allocationId; @@ -321,7 +321,7 @@ public PersistentTask(StreamInput in) throws IOException { id = in.readString(); allocationId = in.readLong(); taskName = in.readString(); - params = (Params) in.readOptionalNamedWriteable(PersistentTaskParams.class); + params = (P) in.readOptionalNamedWriteable(PersistentTaskParams.class); status = in.readOptionalNamedWriteable(Task.Status.class); assignment = new Assignment(in.readOptionalString(), in.readString()); allocationIdOnLastStatusUpdate = in.readOptionalLong(); @@ -377,7 +377,7 @@ public String getTaskName() { } @Nullable - public Params getParams() { + public P getParams() { return params; } From a08e2d9e5e60cfe08cf894f940b60e986a14f22d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 19 Apr 2017 15:42:55 -0400 Subject: [PATCH 32/50] Persistent tasks: require allocation id on task completion (#1107) Persistent tasks should verify that completion notification is done for correct version of the task, otherwise a delayed notification from an old node can accidentally close a newly reassigned task. --- .../persistent/AllocatedPersistentTask.java | 12 ++++++---- .../CompletionPersistentTaskAction.java | 15 +++++++++--- .../PersistentTasksClusterService.java | 23 +++++++++++-------- .../PersistentTasksNodeService.java | 23 +++++++++++++++---- .../persistent/PersistentTasksService.java | 5 ++-- .../persistent/PersistentTasksExecutorIT.java | 12 ++++++++++ .../PersistentTasksNodeServiceTests.java | 3 ++- .../RestartPersistentTaskRequestTests.java | 2 +- 8 files changed, 70 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index dee95ac9814ba..131d7727ce022 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -132,7 +132,7 @@ public void markAsFailed(Exception e) { private void completeAndNotifyIfNeeded(@Nullable Exception failure) { State prevState = state.getAndSet(AllocatedPersistentTask.State.COMPLETED); if (prevState == State.COMPLETED) { - logger.warn("attempt to complete task {} in the {} state", getPersistentTaskId(), prevState); + logger.warn("attempt to complete task [{}] with id [{}] in the [{}] state", getAction(), getPersistentTaskId(), prevState); } else { if (failure != null) { logger.warn((Supplier) () -> new ParameterizedMessage( @@ -141,18 +141,20 @@ private void completeAndNotifyIfNeeded(@Nullable Exception failure) { try { this.failure = failure; if (prevState == State.STARTED) { - logger.trace("sending notification for completed task {}", getPersistentTaskId()); - persistentTasksService.sendCompletionNotification(getPersistentTaskId(), failure, new + logger.trace("sending notification for completed task [{}] with id [{}]", getAction(), getPersistentTaskId()); + persistentTasksService.sendCompletionNotification(getPersistentTaskId(), getAllocationId(), failure, new ActionListener>() { @Override public void onResponse(PersistentTasksCustomMetaData.PersistentTask persistentTask) { - logger.trace("notification for task {} was successful", getId()); + logger.trace("notification for task [{}] with id [{}] was successful", getAction(), + getPersistentTaskId()); } @Override public void onFailure(Exception e) { logger.warn((Supplier) () -> - new ParameterizedMessage("notification for task {} failed", getPersistentTaskId()), e); + new ParameterizedMessage("notification for task [{}] with id [{}] failed", + getAction(), getPersistentTaskId()), e); } }); } diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java index f8042b97d0b78..c4bffeeb44d22 100644 --- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -75,19 +75,23 @@ public static class Request extends MasterNodeRequest { private Exception exception; + private long allocationId = -1; + public Request() { } - public Request(String taskId, Exception exception) { + public Request(String taskId, long allocationId, Exception exception) { this.taskId = taskId; this.exception = exception; + this.allocationId = allocationId; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); taskId = in.readString(); + allocationId = in.readLong(); exception = in.readException(); } @@ -95,6 +99,7 @@ public void readFrom(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(taskId); + out.writeLong(allocationId); out.writeException(exception); } @@ -104,6 +109,9 @@ public ActionRequestValidationException validate() { if (taskId == null) { validationException = addValidationError("task id is missing", validationException); } + if (allocationId < 0) { + validationException = addValidationError("allocation id is negative or missing", validationException); + } return validationException; } @@ -113,12 +121,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; return Objects.equals(taskId, request.taskId) && + allocationId == request.allocationId && Objects.equals(exception, request.exception); } @Override public int hashCode() { - return Objects.hash(taskId, exception); + return Objects.hash(taskId, allocationId, exception); } } @@ -163,7 +172,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) @Override protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { - persistentTasksClusterService.completePersistentTask(request.taskId, request.exception, + persistentTasksClusterService.completePersistentTask(request.taskId, request.allocationId, request.exception, new ActionListener>() { @Override public void onResponse(PersistentTask task) { diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 57b370398f1fc..9bed14d327409 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -59,7 +59,7 @@ public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorR * Creates a new persistent task on master node * * @param action the action name - * @param params params + * @param params params * @param listener the listener that will be called when task is started */ public void createPersistentTask(String taskId, String action, @Nullable Params params, @@ -99,11 +99,12 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS /** * Restarts a record about a running persistent task from cluster state * - * @param id the id of a persistent task - * @param failure the reason for restarting the task or null if the task completed successfully - * @param listener the listener that will be called when task is removed + * @param id the id of the persistent task + * @param allocationId the allocation id of the persistent task + * @param failure the reason for restarting the task or null if the task completed successfully + * @param listener the listener that will be called when task is removed */ - public void completePersistentTask(String id, Exception failure, ActionListener> listener) { + public void completePersistentTask(String id, long allocationId, Exception failure, ActionListener> listener) { final String source; if (failure != null) { logger.warn("persistent task " + id + " failed", failure); @@ -115,13 +116,17 @@ public void completePersistentTask(String id, Exception failure, ActionListener< @Override public ClusterState execute(ClusterState currentState) throws Exception { PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); - if (tasksInProgress.hasTask(id)) { + if (tasksInProgress.hasTask(id, allocationId)) { tasksInProgress.finishTask(id); return update(currentState, tasksInProgress); } else { - // we don't send the error message back to the caller becase that would cause an infinite loop of notifications - logger.warn("The task {} wasn't found, status is not updated", id); - return currentState; + if (tasksInProgress.hasTask(id)) { + logger.warn("The task [{}] with id [{}] was found but it has a different allocation id [{}], status is not updated", + PersistentTasksCustomMetaData.getTaskWithId(currentState, id).getTaskName(), id, allocationId); + } else { + logger.warn("The task [{}] wasn't found, status is not updated", id); + } + throw new ResourceNotFoundException("the task with id [" + id + "] and allocation id [" + allocationId + "] not found"); } } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index 4222c49107f76..111041fcd8b13 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskAwareRequest; import org.elasticsearch.tasks.TaskId; @@ -70,6 +71,11 @@ public PersistentTasksNodeService(Settings settings, @Override public void clusterChanged(ClusterChangedEvent event) { + if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + // wait until the gateway has recovered from disk, otherwise if the only master restarts + // we start cancelling all local tasks before cluster has a chance to recover. + return; + } PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); PersistentTasksCustomMetaData previousTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); @@ -120,11 +126,14 @@ public void clusterChanged(ClusterChangedEvent event) { AllocatedPersistentTask task = runningTasks.get(id); if (task.getState() == AllocatedPersistentTask.State.COMPLETED) { // Result was sent to the caller and the caller acknowledged acceptance of the result + logger.trace("Found completed persistent task [{}] with id [{}] and allocation id [{}] - removing", + task.getAction(), task.getPersistentTaskId(), task.getAllocationId()); runningTasks.remove(id); } else { // task is running locally, but master doesn't know about it - that means that the persistent task was removed // cancel the task without notifying master - logger.trace("Found unregistered persistent task with id {} - cancelling ", id); + logger.trace("Found unregistered persistent task [{}] with id [{}] and allocation id [{}] - cancelling", + task.getAction(), task.getPersistentTaskId(), task.getAllocationId()); cancelTask(id); } } @@ -160,6 +169,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId) boolean processed = false; try { task.init(persistentTasksService, taskManager, logger, taskInProgress.getId(), taskInProgress.getAllocationId()); + logger.trace("Persistent task [{}] with id [{}] and allocation id [{}] was created", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()); try { runningTasks.put(taskInProgress.getAllocationId(), task); nodePersistentTasksExecutor.executeTask(taskInProgress.getParams(), task, executor); @@ -171,6 +182,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId) } finally { if (processed == false) { // something went wrong - unregistering task + logger.warn("Persistent task [{}] with id [{}] and allocation id [{}] failed to create", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()); taskManager.unregister(task); } } @@ -187,14 +200,16 @@ private void cancelTask(Long allocationId) { persistentTasksService.sendTaskManagerCancellation(task.getId(), new ActionListener() { @Override public void onResponse(CancelTasksResponse cancelTasksResponse) { - logger.trace("Persistent task with id {} was cancelled", task.getId()); - + logger.trace("Persistent task [{}] with id [{}] and allocation id [{}] was cancelled", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()); } @Override public void onFailure(Exception e) { // There is really nothing we can do in case of failure here - logger.warn((Supplier) () -> new ParameterizedMessage("failed to cancel task {}", task.getPersistentTaskId()), e); + logger.warn((Supplier) () -> + new ParameterizedMessage("failed to cancel task [{}] with id [{}] and allocation id [{}]", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()), e); } }); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index d7788ae5af9a9..cf416623bcbb7 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -73,8 +73,9 @@ public void startPersistentTask(String tas /** * Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure */ - public void sendCompletionNotification(String taskId, Exception failure, ActionListener> listener) { - CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); + public void sendCompletionNotification(String taskId, long allocationId, Exception failure, + ActionListener> listener) { + CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, allocationId, failure); try { client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index 7790de710bf88..b5dd910fa42a9 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -122,6 +122,18 @@ public void testPersistentActionCompletion() throws Exception { assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(allocationId)); assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); assertThat(firstRunningTask.getDescription(), equalTo("id=" + taskId)); + + if (randomBoolean()) { + logger.info("Simulating errant completion notification"); + //try sending completion request with incorrect allocation id + PlainActionFuture> failedCompletionNotificationFuture = new PlainActionFuture<>(); + persistentTasksService.sendCompletionNotification(taskId, Long.MAX_VALUE, null, failedCompletionNotificationFuture); + assertThrows(failedCompletionNotificationFuture, ResourceNotFoundException.class); + // Make sure that the task is still running + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .setDetailed(true).get().getTasks().size(), equalTo(1)); + } + stopOrCancelTask(firstRunningTask.getTaskId()); } diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 95b0ec87669f1..ea9e8f69dcf32 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -184,7 +184,8 @@ public void sendTaskManagerCancellation(long taskId, ActionListener> listener) { + public void sendCompletionNotification(String taskId, long allocationId, Exception failure, + ActionListener> listener) { fail("Shouldn't be called during Cluster State cancellation"); } }; diff --git a/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java index 3fc8f80490359..3ce29d543d4b9 100644 --- a/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java @@ -25,7 +25,7 @@ public class RestartPersistentTaskRequestTests extends AbstractStreamableTestCas @Override protected Request createTestInstance() { - return new Request(randomAlphaOfLength(10), null); + return new Request(randomAlphaOfLength(10), randomLong(), null); } @Override From 44ea5d6b3e367271c384c15373db577aedaa6010 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 3 Apr 2017 17:46:31 +0200 Subject: [PATCH 33/50] Separate publishing from applying cluster states Companion commit to elastic/elasticsearch#24236 --- .../PersistentTasksNodeServiceTests.java | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index ea9e8f69dcf32..02d44d4eb1f07 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -23,13 +23,13 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -59,25 +59,21 @@ public class PersistentTasksNodeServiceTests extends ESTestCase { - private ClusterService createClusterService() { - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - return new ClusterService(Settings.builder().put("cluster.name", "PersistentActionExecutorTests").build(), - clusterSettings, null, () -> new DiscoveryNode(UUIDs.randomBase64UUID(), buildNewFakeTransportAddress(), - Version.CURRENT)); - } - - private DiscoveryNodes createTestNodes(int nonLocalNodesCount, Settings settings) { + private ClusterState createInitialClusterState(int nonLocalNodesCount, Settings settings) { + ClusterState.Builder state = ClusterState.builder(new ClusterName("PersistentActionExecutorTests")); + state.metaData(MetaData.builder().generateClusterUuidIfNeeded()); + state.routingTable(RoutingTable.builder().build()); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); nodes.add(DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), "this_node")); for (int i = 0; i < nonLocalNodesCount; i++) { nodes.add(new DiscoveryNode("other_node_" + i, buildNewFakeTransportAddress(), Version.CURRENT)); } nodes.localNodeId("this_node"); - return nodes.build(); + state.nodes(nodes); + return state.build(); } public void testStartTask() throws Exception { - ClusterService clusterService = createClusterService(); PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); @@ -95,8 +91,7 @@ public void testStartTask() throws Exception { PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, registry, new TaskManager(Settings.EMPTY), executor); - ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) - .build(); + ClusterState state = createInitialClusterState(nonLocalNodesCount, Settings.EMPTY); PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); boolean added = false; @@ -173,7 +168,6 @@ public void testStartTask() throws Exception { } public void testTaskCancellation() { - ClusterService clusterService = createClusterService(); AtomicLong capturedTaskId = new AtomicLong(); AtomicReference> capturedListener = new AtomicReference<>(); PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null, null) { @@ -202,8 +196,7 @@ public void sendCompletionNotification(String taskId, long allocationId, Excepti PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, registry, taskManager, executor); - ClusterState state = ClusterState.builder(clusterService.state()).nodes(createTestNodes(nonLocalNodesCount, Settings.EMPTY)) - .build(); + ClusterState state = createInitialClusterState(nonLocalNodesCount, Settings.EMPTY); ClusterState newClusterState = state; // Allocate first task From e69317b24bc2d27fe2b98b0f04f25440191ae7dd Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Sat, 29 Apr 2017 10:49:12 +0200 Subject: [PATCH 34/50] Don't call ClusterService.state() in a ClusterStateUpdateTask The current state is readily available as a parameter --- .../elasticsearch/persistent/PersistentTasksClusterService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index 9bed14d327409..24d8c5f7be31a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -71,7 +71,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (builder.hasTask(taskId)) { throw new ResourceAlreadyExistsException("task with id {" + taskId + "} already exist"); } - validate(action, clusterService.state(), params); + validate(action, currentState, params); final Assignment assignment; assignment = getAssignement(action, currentState, params); return update(currentState, builder.addTask(taskId, action, params, assignment)); From 292e383d2c7a4dd59258fc3e141001d2699699fb Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 17 May 2017 17:28:35 +0200 Subject: [PATCH 35/50] Fix static / version based BWC tests (#1456) With the leniency in Version.java we missed to really setup BWC testing for static indices. This change brings back the testing and adds missing bwc indices. Relates to elastic/elasticsearch#24732 --- .../elasticsearch/persistent/PersistentTasksCustomMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index c24dd6c1b2853..c827a09582f81 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -196,7 +196,7 @@ public long getNumberOfTasksOnNode(String nodeId, String taskName) { @Override public Version getMinimalSupportedVersion() { - return Version.V_5_4_0_UNRELEASED; + return Version.V_5_4_0; } @Override From 614aef2527044e36a1697b2a01cbf962be46a853 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 22 May 2017 08:48:33 +0200 Subject: [PATCH 36/50] Pass down the provided timeout. --- .../org/elasticsearch/persistent/PersistentTasksService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index cf416623bcbb7..220f55aff333e 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -180,7 +180,7 @@ public void onClusterServiceClose() { public void onTimeout(TimeValue timeout) { listener.onFailure(new IllegalStateException("timed out after " + timeout)); } - }, clusterState -> predicate.test(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE))); + }, clusterState -> predicate.test(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)), timeout); } } From 1cef531165e9488a9032596ae7b907d0e1fcc2c1 Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Fri, 2 Jun 2017 10:01:21 -0400 Subject: [PATCH 37/50] Always Accumulate Transport Exceptions (#1619) This is the x-pack side of the removal of `accumulateExceptions()` for both `TransportNodesAction` and `TransportTasksAction`. There are occasional, random failures that occur during API calls that are silently ignored from the caller's perspective, which also leads to weird API responses that have no response and also no errors, which is obviously untrue. --- .../elasticsearch/persistent/TestPersistentTasksPlugin.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 5f3b6c13aaa4b..95917f205c2bd 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -546,10 +546,6 @@ protected void taskOperation(TestTasksRequest request, TestTask task, ActionList listener.onResponse(new TestTaskResponse()); } - @Override - protected boolean accumulateExceptions() { - return false; - } } From 0d50f9c6a9a4de1bd3caaf351bc99bd630c8d0be Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 19 Jun 2017 16:40:32 +0100 Subject: [PATCH 38/50] Call initialising constructor of BaseTasksRequest (#1771) --- .../org/elasticsearch/persistent/TestPersistentTasksPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 95917f205c2bd..c2ce50ed73c8f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -491,7 +491,7 @@ public static class TestTasksResponse extends BaseTasksResponse { private List tasks; public TestTasksResponse() { - + super(null, null); } public TestTasksResponse(List tasks, List taskFailures, From ffdb05e48e0d57e59509e7ff06939f8428f05374 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 26 Jul 2017 08:40:22 -0400 Subject: [PATCH 39/50] Persistent Tasks: remove unused isCurrentStatus method (#2076) Removes a method that is no longer used in production code. Relates to #957 --- .../persistent/PersistentTasksCustomMetaData.java | 8 -------- .../persistent/PersistentTasksExecutorIT.java | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index c827a09582f81..f6ef778db0747 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -406,14 +406,6 @@ public Status getStatus() { return status; } - /** - * @return Whether the task status isn't stale. When a task gets unassigned from the executor node or assigned - * to a new executor node and the status hasn't been updated then the task status is stale. - */ - public boolean isCurrentStatus() { - return allocationIdOnLastStatusUpdate != null && allocationIdOnLastStatusUpdate == allocationId; - } - @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params xParams) throws IOException { builder.startObject(); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java index b5dd910fa42a9..91ccd8a37f06b 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -202,7 +202,7 @@ public void testPersistentActionStatusUpdate() throws Exception { int finalI = i; WaitForPersistentTaskStatusFuture future1 = new WaitForPersistentTaskStatusFuture<>(); persistentTasksService.waitForPersistentTaskStatus(taskId, - task -> task != null && task.isCurrentStatus() && task.getStatus().toString() != null && + task -> task != null && task.getStatus() != null && task.getStatus().toString() != null && task.getStatus().toString().equals("{\"phase\":\"phase " + (finalI + 1) + "\"}"), TimeValue.timeValueSeconds(10), future1); assertThat(future1.get().getId(), equalTo(taskId)); From b5f281386a09711077b2269d1bec1a740bc0f6e4 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 28 Jul 2017 11:23:52 +0200 Subject: [PATCH 40/50] Move tribe to a module (#2088) Companion PR to elastic/elasticsearch#25778 --- .../elasticsearch/persistent/TestPersistentTasksPlugin.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index c2ce50ed73c8f..12773c4b6f7ab 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -51,6 +51,8 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; @@ -101,7 +103,8 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { @Override public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, - NamedXContentRegistry xContentRegistry) { + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { InternalClient internalClient = new InternalClient(Settings.EMPTY, threadPool, client); PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, internalClient); TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, clusterService); From 65ce2276ebe46789b81ff3cd19eaad3f2f2638e6 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 2 Aug 2017 08:43:09 +0200 Subject: [PATCH 41/50] Adapt to upstream changes made to AbstractStreamableXContentTestCase (#2117) --- .../PersistentTasksCustomMetaDataTests.java | 27 +++++-------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java index ea33d6cfe9f5e..7e731884dda41 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -25,12 +25,12 @@ import org.elasticsearch.cluster.metadata.MetaData.Custom; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -40,8 +40,8 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Builder; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; -import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import java.io.IOException; import java.util.ArrayList; @@ -127,27 +127,13 @@ protected PersistentTasksCustomMetaData doParseInstance(XContentParser parser) t return PersistentTasksCustomMetaData.fromXContent(parser); } +/* @Override protected XContentBuilder toXContent(Custom instance, XContentType contentType) throws IOException { return toXContent(instance, contentType, new ToXContent.MapParams( Collections.singletonMap(MetaData.CONTEXT_MODE_PARAM, MetaData.XContentContext.API.toString()))); } - - protected XContentBuilder toXContent(Custom instance, XContentType contentType, ToXContent.MapParams params) throws IOException { - // We need all attribute to be serialized/de-serialized for testing - XContentBuilder builder = XContentFactory.contentBuilder(contentType); - if (randomBoolean()) { - builder.prettyPrint(); - } - if (instance.isFragment()) { - builder.startObject(); - } - instance.toXContent(builder, params); - if (instance.isFragment()) { - builder.endObject(); - } - return builder; - } +*/ private String addRandomTask(Builder builder) { String taskId = UUIDs.base64UUID(); @@ -179,10 +165,9 @@ public void testSerializationContext() throws Exception { Collections.singletonMap(MetaData.CONTEXT_MODE_PARAM, randomFrom(CONTEXT_MODE_SNAPSHOT, CONTEXT_MODE_GATEWAY))); XContentType xContentType = randomFrom(XContentType.values()); - XContentBuilder builder = toXContent(testInstance, xContentType, params); - XContentBuilder shuffled = shuffleXContent(builder); + BytesReference shuffled = toShuffledXContent(testInstance, xContentType, params, false); - XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled.bytes()); + XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled); PersistentTasksCustomMetaData newInstance = doParseInstance(parser); assertNotSame(newInstance, testInstance); From b0de3c38d6322892fa976e808a273a740ea6de10 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 17 Aug 2017 11:16:56 +0100 Subject: [PATCH 42/50] Moves more classes over to ToXContentObject/Fragment (#2283) --- .../persistent/PersistentTasksCustomMetaData.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java index f6ef778db0747..237157e44c43c 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.tasks.Task; @@ -266,7 +267,7 @@ public String toString() { /** * A record that represents a single running persistent task */ - public static class PersistentTask

implements Writeable, ToXContent { + public static class PersistentTask

implements Writeable, ToXContentObject { private final String id; private final long allocationId; private final String taskName; From 7313ad5b29059eb381efd20a6c77c472f527d71c Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 17 Aug 2017 14:54:31 +0100 Subject: [PATCH 43/50] Make AllocatedPersistentTask members volatile (#2297) These members are default initialized on contruction and then set by the init() method. It's possible that another thread accessing the object after init() is called could still see the null/0 values, depending on how the compiler optimizes the code. --- .../persistent/AllocatedPersistentTask.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index 131d7727ce022..21d3fa7e95738 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -36,16 +36,16 @@ * Represents a executor node operation that corresponds to a persistent task */ public class AllocatedPersistentTask extends CancellableTask { - private String persistentTaskId; - private long allocationId; + private volatile String persistentTaskId; + private volatile long allocationId; private final AtomicReference state; @Nullable - private Exception failure; + private volatile Exception failure; - private PersistentTasksService persistentTasksService; - private Logger logger; - private TaskManager taskManager; + private volatile PersistentTasksService persistentTasksService; + private volatile Logger logger; + private volatile TaskManager taskManager; public AllocatedPersistentTask(long id, String type, String action, String description, TaskId parentTask) { From 1c489ee8671dadd54110b9f8290565f92c973927 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 23 Aug 2017 08:17:28 +0100 Subject: [PATCH 44/50] Refactor/to x content fragments2 (#2329) * Moves more classes over to ToXContentObject/Fragment * Removes ToXContentToBytes * Removes ToXContent from Enums * review comment fix * slight change to use XContantHelper --- .../org/elasticsearch/persistent/PersistentTaskParams.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java index ecf5c7c318978..a475a7cde174a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java @@ -20,11 +20,11 @@ package org.elasticsearch.persistent; import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; /** * Parameters used to start persistent task */ -public interface PersistentTaskParams extends NamedWriteable, ToXContent { +public interface PersistentTaskParams extends NamedWriteable, ToXContentObject { } From 4dd69951f346bd7ea4fa068f9d80ad89542db340 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 24 Oct 2017 10:20:33 +0200 Subject: [PATCH 45/50] Make the persistent task status available to PersistentTasksExecutor.nodeOperation(...) method --- .../NodePersistentTasksExecutor.java | 4 +- .../persistent/PersistentTasksExecutor.java | 5 +- .../PersistentTasksNodeService.java | 2 +- .../PersistentTasksNodeServiceTests.java | 47 +++++++++++++++++-- .../persistent/TestPersistentTasksPlugin.java | 8 ++-- 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java index 7efbc92b27809..efed0aef9b807 100644 --- a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; /** @@ -35,6 +36,7 @@ public NodePersistentTasksExecutor(ThreadPool threadPool) { } public void executeTask(@Nullable Params params, + @Nullable Task.Status status, AllocatedPersistentTask task, PersistentTasksExecutor executor) { threadPool.executor(executor.getExecutor()).execute(new AbstractRunnable() { @@ -47,7 +49,7 @@ public void onFailure(Exception e) { @Override protected void doRun() throws Exception { try { - executor.nodeOperation(task, params); + executor.nodeOperation(task, params, status); } catch (Exception ex) { task.markAsFailed(ex); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index d8f5d93126a60..0fa7f2dcba0e7 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; @@ -115,10 +116,10 @@ protected String getDescription(PersistentTask taskInProgress) { /** * This operation will be executed on the executor node. *

- * NOTE: The nodeOperation has to throws an exception, trigger task.markAsCompleted() or task.completeAndNotifyIfNeeded() methods to + * NOTE: The nodeOperation has to throw an exception, trigger task.markAsCompleted() or task.completeAndNotifyIfNeeded() methods to * indicate that the persistent task has finished. */ - protected abstract void nodeOperation(AllocatedPersistentTask task, @Nullable Params params); + protected abstract void nodeOperation(AllocatedPersistentTask task, @Nullable Params params, @Nullable Task.Status status); public String getExecutor() { return executor; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index 111041fcd8b13..cfdf221e68186 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -173,7 +173,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId) task.getPersistentTaskId(), task.getAllocationId()); try { runningTasks.put(taskInProgress.getAllocationId(), task); - nodePersistentTasksExecutor.executeTask(taskInProgress.getParams(), task, executor); + nodePersistentTasksExecutor.executeTask(taskInProgress.getParams(), taskInProgress.getStatus(), task, executor); } catch (Exception e) { // Submit task failure task.markAsFailed(e); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 02d44d4eb1f07..7f36d6c890259 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -49,12 +49,14 @@ import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.core.IsEqual.equalTo; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class PersistentTasksNodeServiceTests extends ESTestCase { @@ -167,6 +169,41 @@ public void testStartTask() throws Exception { } + public void testParamsStatusAndNodeTaskAreDelegated() throws Exception { + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); + TaskId parentId = new TaskId("cluster", 1); + AllocatedPersistentTask nodeTask = new TestPersistentTasksPlugin.TestTask(0, "persistent", "test", "", parentId); + when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any())).thenReturn(nodeTask); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); + + MockExecutor executor = new MockExecutor(); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, new TaskManager(Settings.EMPTY), executor); + + ClusterState state = createInitialClusterState(1, Settings.EMPTY); + + Task.Status status = new TestPersistentTasksPlugin.Status("_test_phase"); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); + String taskId = UUIDs.base64UUID(); + TestParams taskParams = new TestParams("other_0"); + tasks.addTask(taskId, TestPersistentTasksExecutor.NAME, taskParams, + new Assignment("this_node", "test assignment on other node")); + tasks.updateTaskStatus(taskId, status); + MetaData.Builder metaData = MetaData.builder(state.metaData()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); + ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); + + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + assertThat(executor.size(), equalTo(1)); + assertThat(executor.get(0).params, sameInstance(taskParams)); + assertThat(executor.get(0).status, sameInstance(status)); + assertThat(executor.get(0).task, sameInstance(nodeTask)); + } + public void testTaskCancellation() { AtomicLong capturedTaskId = new AtomicLong(); AtomicReference> capturedListener = new AtomicReference<>(); @@ -271,11 +308,13 @@ private ClusterState removeTask(ClusterState state, String taskId) { private class Execution { private final PersistentTaskParams params; private final AllocatedPersistentTask task; + private final Task.Status status; private final PersistentTasksExecutor holder; - Execution(PersistentTaskParams params, AllocatedPersistentTask task, PersistentTasksExecutor holder) { + Execution(PersistentTaskParams params, AllocatedPersistentTask task, Task.Status status, PersistentTasksExecutor holder) { this.params = params; this.task = task; + this.status = status; this.holder = holder; } } @@ -288,9 +327,11 @@ private class MockExecutor extends NodePersistentTasksExecutor { } @Override - public void executeTask(Params params, AllocatedPersistentTask task, + public void executeTask(Params params, + Task.Status status, + AllocatedPersistentTask task, PersistentTasksExecutor executor) { - executions.add(new Execution(params, task, executor)); + executions.add(new Execution(params, task, status, executor)); } public Execution get(int i) { diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 12773c4b6f7ab..b024359e209c6 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -323,7 +323,7 @@ public Assignment getAssignment(TestParams params, ClusterState clusterState) { } @Override - protected void nodeOperation(AllocatedPersistentTask task, TestParams params) { + protected void nodeOperation(AllocatedPersistentTask task, TestParams params, Task.Status status) { logger.info("started node operation for the task {}", task); try { TestTask testTask = (TestTask) task; @@ -346,9 +346,9 @@ protected void nodeOperation(AllocatedPersistentTask task, TestParams params) { } else if ("update_status".equals(testTask.getOperation())) { testTask.setOperation(null); CountDownLatch latch = new CountDownLatch(1); - Status status = new Status("phase " + phase.incrementAndGet()); - logger.info("updating the task status to {}", status); - task.updatePersistentStatus(status, new ActionListener>() { + Status newStatus = new Status("phase " + phase.incrementAndGet()); + logger.info("updating the task status to {}", newStatus); + task.updatePersistentStatus(newStatus, new ActionListener>() { @Override public void onResponse(PersistentTask persistentTask) { logger.info("updating was successful"); From 8521b2d11ea8713411090f8d317ee77840b0b2b8 Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Wed, 22 Nov 2017 08:35:18 -0700 Subject: [PATCH 46/50] Remove InternalClient and InternalSecurityClient (#3054) This change removes the InternalClient and the InternalSecurityClient. These are replaced with usage of the ThreadContext and a transient value, `action.origin`, to indicate which component the request came from. The security code has been updated to look for this value and ensure the request is executed as the proper user. This work comes from #2808 where @s1monw suggested that we do this. While working on this, I came across index template registries and rather than updating them to use the new method, I replaced the ML one with the template upgrade framework so that we could remove this template registry. The watcher template registry is still needed as the template must be updated for rolling upgrades to work (see #2950). --- .../persistent/PersistentTasksService.java | 26 +++++++++++-------- .../persistent/TestPersistentTasksPlugin.java | 4 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 220f55aff333e..4958e6183404a 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -34,20 +35,22 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; -import org.elasticsearch.security.InternalClient; import java.util.function.Predicate; +import static org.elasticsearch.ClientHelper.PERSISTENT_TASK_ORIGIN; +import static org.elasticsearch.ClientHelper.executeAsyncWithOrigin; + /** * This service is used by persistent actions to propagate changes in the action state and notify about completion */ public class PersistentTasksService extends AbstractComponent { - private final InternalClient client; + private final Client client; private final ClusterService clusterService; private final ThreadPool threadPool; - public PersistentTasksService(Settings settings, ClusterService clusterService, ThreadPool threadPool, InternalClient client) { + public PersistentTasksService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { super(settings); this.client = client; this.clusterService = clusterService; @@ -63,8 +66,8 @@ public void startPersistentTask(String tas StartPersistentTaskAction.Request createPersistentActionRequest = new StartPersistentTaskAction.Request(taskId, taskName, params); try { - client.execute(StartPersistentTaskAction.INSTANCE, createPersistentActionRequest, ActionListener.wrap( - o -> listener.onResponse((PersistentTask) o.getTask()), listener::onFailure)); + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, StartPersistentTaskAction.INSTANCE, createPersistentActionRequest, + ActionListener.wrap(o -> listener.onResponse((PersistentTask) o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); } @@ -77,7 +80,7 @@ public void sendCompletionNotification(String taskId, long allocationId, Excepti ActionListener> listener) { CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, allocationId, failure); try { - client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, CompletionPersistentTaskAction.INSTANCE, restartRequest, ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); @@ -93,7 +96,8 @@ void sendTaskManagerCancellation(long taskId, ActionListener listener.onResponse(o.getTask()), listener::onFailure)); + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, + ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); } @@ -122,8 +126,8 @@ void updateStatus(String taskId, long allocationId, Task.Status status, ActionLi public void cancelPersistentTask(String taskId, ActionListener> listener) { RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId); try { - client.execute(RemovePersistentTaskAction.INSTANCE, removeRequest, ActionListener.wrap(o -> listener.onResponse(o.getTask()), - listener::onFailure)); + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, RemovePersistentTaskAction.INSTANCE, removeRequest, + ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); } catch (Exception e) { listener.onFailure(e); } diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index b024359e209c6..5204a64eacbcb 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -64,7 +64,6 @@ import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; -import org.elasticsearch.security.InternalClient; import java.io.IOException; import java.util.ArrayList; @@ -105,8 +104,7 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - InternalClient internalClient = new InternalClient(Settings.EMPTY, threadPool, client); - PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, internalClient); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, client); TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, clusterService); PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(testPersistentAction)); From 41071e4711a8c60f05d0cd91ce08c3ba2787afe0 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 12 Jan 2018 15:34:28 -0500 Subject: [PATCH 47/50] Add adding ability to associate an ID with tasks. Persistent tasks portion of elastic/elasticsearch#23250 --- .../persistent/AllocatedPersistentTask.java | 6 ++- .../persistent/PersistentTasksExecutor.java | 5 ++- .../PersistentTasksNodeService.java | 4 +- .../PersistentTasksNodeServiceTests.java | 40 ++++++++++++++----- .../persistent/TestPersistentTasksPlugin.java | 9 +++-- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index 21d3fa7e95738..180e8f111041b 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -30,6 +30,7 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; /** @@ -48,8 +49,9 @@ public class AllocatedPersistentTask extends CancellableTask { private volatile TaskManager taskManager; - public AllocatedPersistentTask(long id, String type, String action, String description, TaskId parentTask) { - super(id, type, action, description, parentTask); + public AllocatedPersistentTask(long id, String type, String action, String description, TaskId parentTask, + Map headers) { + super(id, type, action, description, parentTask, headers); this.state = new AtomicReference<>(State.STARTED); } diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java index 0fa7f2dcba0e7..ed61ad5805391 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -29,6 +29,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import java.util.Map; import java.util.function.Predicate; /** @@ -102,8 +103,8 @@ public void validate(Params params, ClusterState clusterState) { * Creates a AllocatedPersistentTask for communicating with task manager */ protected AllocatedPersistentTask createTask(long id, String type, String action, TaskId parentTaskId, - PersistentTask taskInProgress) { - return new AllocatedPersistentTask(id, type, action, getDescription(taskInProgress), parentTaskId); + PersistentTask taskInProgress, Map headers) { + return new AllocatedPersistentTask(id, type, action, getDescription(taskInProgress), parentTaskId, headers); } /** diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java index cfdf221e68186..e53834d6f4655 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -160,8 +160,8 @@ public TaskId getParentTask() { } @Override - public Task createTask(long id, String type, String action, TaskId parentTaskId) { - return executor.createTask(id, type, action, parentTaskId, taskInProgress); + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return executor.createTask(id, type, action, parentTaskId, taskInProgress, headers); } }; AllocatedPersistentTask task = (AllocatedPersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]", diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index 7f36d6c890259..f9a70637e502f 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -35,11 +35,14 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.ArrayList; @@ -61,6 +64,23 @@ public class PersistentTasksNodeServiceTests extends ESTestCase { + private ThreadPool threadPool; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + threadPool = new TestThreadPool(getClass().getName()); + } + + + @Override + @After + public void tearDown() throws Exception { + terminate(threadPool); + super.tearDown(); + } + private ClusterState createInitialClusterState(int nonLocalNodesCount, Settings settings) { ClusterState.Builder state = ClusterState.builder(new ClusterName("PersistentActionExecutorTests")); state.metaData(MetaData.builder().generateClusterUuidIfNeeded()); @@ -84,14 +104,14 @@ public void testStartTask() throws Exception { // need to account for 5 original tasks on each node and their relocations for (int i = 0; i < (nonLocalNodesCount + 1) * 10; i++) { TaskId parentId = new TaskId("cluster", i); - when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any())).thenReturn( - new TestPersistentTasksPlugin.TestTask(i, "persistent", "test", "", parentId)); + when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any(), any())).thenReturn( + new TestPersistentTasksPlugin.TestTask(i, "persistent", "test", "", parentId, Collections.emptyMap())); } PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); MockExecutor executor = new MockExecutor(); PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, - registry, new TaskManager(Settings.EMPTY), executor); + registry, new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), executor); ClusterState state = createInitialClusterState(nonLocalNodesCount, Settings.EMPTY); @@ -175,13 +195,14 @@ public void testParamsStatusAndNodeTaskAreDelegated() throws Exception { when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); TaskId parentId = new TaskId("cluster", 1); - AllocatedPersistentTask nodeTask = new TestPersistentTasksPlugin.TestTask(0, "persistent", "test", "", parentId); - when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any())).thenReturn(nodeTask); + AllocatedPersistentTask nodeTask = + new TestPersistentTasksPlugin.TestTask(0, "persistent", "test", "", parentId, Collections.emptyMap()); + when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any(), any())).thenReturn(nodeTask); PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); MockExecutor executor = new MockExecutor(); PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, - registry, new TaskManager(Settings.EMPTY), executor); + registry, new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), executor); ClusterState state = createInitialClusterState(1, Settings.EMPTY); @@ -223,13 +244,14 @@ public void sendCompletionNotification(String taskId, long allocationId, Excepti @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); when(action.getTaskName()).thenReturn("test"); - when(action.createTask(anyLong(), anyString(), anyString(), any(), any())) - .thenReturn(new TestPersistentTasksPlugin.TestTask(1, "persistent", "test", "", new TaskId("cluster", 1))); + when(action.createTask(anyLong(), anyString(), anyString(), any(), any(), any())) + .thenReturn(new TestPersistentTasksPlugin.TestTask(1, "persistent", "test", "", new TaskId("cluster", 1), + Collections.emptyMap())); PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); int nonLocalNodesCount = randomInt(10); MockExecutor executor = new MockExecutor(); - TaskManager taskManager = new TaskManager(Settings.EMPTY); + TaskManager taskManager = new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()); PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, registry, taskManager, executor); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 5204a64eacbcb..ba8e2337fd742 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -71,6 +71,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -384,8 +385,8 @@ public void onFailure(Exception e) { @Override protected AllocatedPersistentTask createTask(long id, String type, String action, TaskId parentTaskId, - PersistentTask task) { - return new TestTask(id, type, action, getDescription(task), parentTaskId); + PersistentTask task, Map headers) { + return new TestTask(id, type, action, getDescription(task), parentTaskId, headers); } } @@ -413,8 +414,8 @@ public TestTasksRequestBuilder newRequestBuilder(ElasticsearchClient client) { public static class TestTask extends AllocatedPersistentTask { private volatile String operation; - public TestTask(long id, String type, String action, String description, TaskId parentTask) { - super(id, type, action, description, parentTask); + public TestTask(long id, String type, String action, String description, TaskId parentTask, Map headers) { + super(id, type, action, description, parentTask, headers); } public String getOperation() { From cc16f9d9c928fd08b101da7f94b98d51c3800e5a Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 23 Jan 2018 16:55:08 +0000 Subject: [PATCH 48/50] Added AllocatedPersistentTask#waitForPersistentTaskStatus(...) that delegates to PersistentTasksService#waitForPersistentTaskStatus(...) This allows persistent tasks executor implementations to not have an instance of PersistentTasksService. --- .../persistent/AllocatedPersistentTask.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java index 180e8f111041b..a0572f93e5e00 100644 --- a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; @@ -32,6 +33,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; /** * Represents a executor node operation that corresponds to a persistent task @@ -118,6 +120,15 @@ public enum State { COMPLETED // the task is done running and trying to notify caller } + /** + * Waits for this persistent task to have the desired state. + */ + public void waitForPersistentTaskStatus(Predicate> predicate, + @Nullable TimeValue timeout, + PersistentTasksService.WaitForPersistentTaskStatusListener listener) { + persistentTasksService.waitForPersistentTaskStatus(persistentTaskId, predicate, timeout, listener); + } + public void markAsCompleted() { completeAndNotifyIfNeeded(null); } From 07e727c769d0be2bc2447b9a23d1aebef7404d31 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 25 Jan 2018 14:59:49 +0100 Subject: [PATCH 49/50] Removed ClientHelper dependency from PersistentTasksService. --- .../persistent/PersistentTasksService.java | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java index 4958e6183404a..2b656dd219cf9 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -18,9 +18,14 @@ */ package org.elasticsearch.persistent; +import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; @@ -30,16 +35,16 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import java.util.function.BiConsumer; import java.util.function.Predicate; - -import static org.elasticsearch.ClientHelper.PERSISTENT_TASK_ORIGIN; -import static org.elasticsearch.ClientHelper.executeAsyncWithOrigin; +import java.util.function.Supplier; /** * This service is used by persistent actions to propagate changes in the action state and notify about completion @@ -194,4 +199,40 @@ default void onTimeout(TimeValue timeout) { onFailure(new IllegalStateException("timed out after " + timeout)); } } + + private static final String ACTION_ORIGIN_TRANSIENT_NAME = "action.origin"; + private static final String PERSISTENT_TASK_ORIGIN = "persistent_tasks"; + + /** + * Executes a consumer after setting the origin and wrapping the listener so that the proper context is restored + */ + public static void executeAsyncWithOrigin( + ThreadContext threadContext, String origin, Request request, ActionListener listener, + BiConsumer> consumer) { + final Supplier supplier = threadContext.newRestorableContext(false); + try (ThreadContext.StoredContext ignore = stashWithOrigin(threadContext, origin)) { + consumer.accept(request, new ContextPreservingActionListener<>(supplier, listener)); + } + } + /** + * Executes an asynchronous action using the provided client. The origin is set in the context and the listener + * is wrapped to ensure the proper context is restored + */ + public static > void executeAsyncWithOrigin( + Client client, String origin, Action action, Request request, + ActionListener listener) { + final ThreadContext threadContext = client.threadPool().getThreadContext(); + final Supplier supplier = threadContext.newRestorableContext(false); + try (ThreadContext.StoredContext ignore = stashWithOrigin(threadContext, origin)) { + client.execute(action, request, new ContextPreservingActionListener<>(supplier, listener)); + } + } + + public static ThreadContext.StoredContext stashWithOrigin(ThreadContext threadContext, String origin) { + final ThreadContext.StoredContext storedContext = threadContext.stashContext(); + threadContext.putTransient(ACTION_ORIGIN_TRANSIENT_NAME, origin); + return storedContext; + } + } From 592eedbf495e95b4728b370b705a2409b41af6b9 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 25 Jan 2018 21:40:47 +0100 Subject: [PATCH 50/50] Make persistent tasks work. Made persistent tasks executors pluggable. --- .../elasticsearch/action/ActionModule.java | 11 +++++- .../java/org/elasticsearch/node/Node.java | 19 +++++++++ .../plugins/PersistentTaskPlugin.java | 39 +++++++++++++++++++ .../persistent/TestPersistentTasksPlugin.java | 26 +++---------- 4 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 872c217f98091..51abf6b0222e1 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -312,9 +312,12 @@ import org.elasticsearch.rest.action.search.RestMultiSearchAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.rest.action.search.RestSearchScrollAction; -import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.usage.UsageService; +import org.elasticsearch.persistent.CompletionPersistentTaskAction; +import org.elasticsearch.persistent.RemovePersistentTaskAction; +import org.elasticsearch.persistent.StartPersistentTaskAction; +import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction; import java.util.ArrayList; import java.util.Collections; @@ -507,6 +510,12 @@ public void reg actionPlugins.stream().flatMap(p -> p.getActions().stream()).forEach(actions::register); + // Persistent tasks: + actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class); + actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class); + actions.register(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class); + actions.register(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class); + return unmodifiableMap(actions.getRegistry()); } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index d0470ea2c42c6..15c0428d25919 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -117,6 +117,7 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.plugins.NetworkPlugin; +import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.RepositoryPlugin; @@ -139,6 +140,10 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.usage.UsageService; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.persistent.PersistentTasksClusterService; +import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.persistent.PersistentTasksExecutorRegistry; +import org.elasticsearch.persistent.PersistentTasksService; import java.io.BufferedWriter; import java.io.Closeable; @@ -461,6 +466,17 @@ protected Node(final Environment environment, Collection threadPool, scriptModule.getScriptService(), bigArrays, searchModule.getFetchPhase(), responseCollectorService); + final List> tasksExecutors = pluginsService + .filterPlugins(PersistentTaskPlugin.class).stream() + .map(p -> p.getPersistentTasksExecutor(clusterService)) + .flatMap(List::stream) + .collect(toList()); + + final PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(settings, tasksExecutors); + final PersistentTasksClusterService persistentTasksClusterService = + new PersistentTasksClusterService(settings, registry, clusterService); + final PersistentTasksService persistentTasksService = new PersistentTasksService(settings, clusterService, threadPool, client); + modules.add(b -> { b.bind(Node.class).toInstance(this); b.bind(NodeService.class).toInstance(nodeService); @@ -504,6 +520,9 @@ protected Node(final Environment environment, Collection } httpBind.accept(b); pluginComponents.stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p)); + b.bind(PersistentTasksService.class).toInstance(persistentTasksService); + b.bind(PersistentTasksClusterService.class).toInstance(persistentTasksClusterService); + b.bind(PersistentTasksExecutorRegistry.class).toInstance(registry); } ); injector = modules.createInjector(); diff --git a/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java b/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java new file mode 100644 index 0000000000000..c402b907ffde6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.plugins; + +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.persistent.PersistentTasksExecutor; + +import java.util.Collections; +import java.util.List; + +/** + * Plugin for registering persistent tasks executors. + */ +public interface PersistentTaskPlugin { + + /** + * Returns additional persistent tasks executors added by this plugin. + */ + default List> getPersistentTasksExecutor(ClusterService clusterService) { + return Collections.emptyList(); + } + +} diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index ba8e2337fd742..ca3e840028cb6 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -54,6 +54,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.tasks.Task; @@ -87,33 +88,16 @@ /** * A plugin that adds a test persistent task. */ -public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin { +public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin, PersistentTaskPlugin { @Override public List> getActions() { - return Arrays.asList( - new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class), - new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), - new ActionHandler<>(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class), - new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), - new ActionHandler<>(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class) - ); + return Collections.singletonList(new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class)); } @Override - public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, ScriptService scriptService, - NamedXContentRegistry xContentRegistry, Environment environment, - NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, clusterService, threadPool, client); - TestPersistentTasksExecutor testPersistentAction = new TestPersistentTasksExecutor(Settings.EMPTY, clusterService); - PersistentTasksExecutorRegistry persistentTasksExecutorRegistry = new PersistentTasksExecutorRegistry(Settings.EMPTY, - Collections.singletonList(testPersistentAction)); - return Arrays.asList( - persistentTasksService, - persistentTasksExecutorRegistry, - new PersistentTasksClusterService(Settings.EMPTY, persistentTasksExecutorRegistry, clusterService) - ); + public List> getPersistentTasksExecutor(ClusterService clusterService) { + return Collections.singletonList(new TestPersistentTasksExecutor(Settings.EMPTY, clusterService)); } @Override