diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 872c217f98091..51abf6b0222e1 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -312,9 +312,12 @@ import org.elasticsearch.rest.action.search.RestMultiSearchAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.rest.action.search.RestSearchScrollAction; -import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.usage.UsageService; +import org.elasticsearch.persistent.CompletionPersistentTaskAction; +import org.elasticsearch.persistent.RemovePersistentTaskAction; +import org.elasticsearch.persistent.StartPersistentTaskAction; +import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction; import java.util.ArrayList; import java.util.Collections; @@ -507,6 +510,12 @@ public void reg actionPlugins.stream().flatMap(p -> p.getActions().stream()).forEach(actions::register); + // Persistent tasks: + actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class); + actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class); + actions.register(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class); + actions.register(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class); + return unmodifiableMap(actions.getRegistry()); } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index d0470ea2c42c6..15c0428d25919 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -117,6 +117,7 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.plugins.NetworkPlugin; +import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.RepositoryPlugin; @@ -139,6 +140,10 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.usage.UsageService; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.persistent.PersistentTasksClusterService; +import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.persistent.PersistentTasksExecutorRegistry; +import org.elasticsearch.persistent.PersistentTasksService; import java.io.BufferedWriter; import java.io.Closeable; @@ -461,6 +466,17 @@ protected Node(final Environment environment, Collection threadPool, scriptModule.getScriptService(), bigArrays, searchModule.getFetchPhase(), responseCollectorService); + final List> tasksExecutors = pluginsService + .filterPlugins(PersistentTaskPlugin.class).stream() + .map(p -> p.getPersistentTasksExecutor(clusterService)) + .flatMap(List::stream) + .collect(toList()); + + final PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(settings, tasksExecutors); + final PersistentTasksClusterService persistentTasksClusterService = + new PersistentTasksClusterService(settings, registry, clusterService); + final PersistentTasksService persistentTasksService = new PersistentTasksService(settings, clusterService, threadPool, client); + modules.add(b -> { b.bind(Node.class).toInstance(this); b.bind(NodeService.class).toInstance(nodeService); @@ -504,6 +520,9 @@ protected Node(final Environment environment, Collection } httpBind.accept(b); pluginComponents.stream().forEach(p -> b.bind((Class) p.getClass()).toInstance(p)); + b.bind(PersistentTasksService.class).toInstance(persistentTasksService); + b.bind(PersistentTasksClusterService.class).toInstance(persistentTasksClusterService); + b.bind(PersistentTasksExecutorRegistry.class).toInstance(registry); } ); injector = modules.createInjector(); diff --git a/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java new file mode 100644 index 0000000000000..a0572f93e5e00 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/AllocatedPersistentTask.java @@ -0,0 +1,179 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; + +/** + * Represents a executor node operation that corresponds to a persistent task + */ +public class AllocatedPersistentTask extends CancellableTask { + private volatile String persistentTaskId; + private volatile long allocationId; + + private final AtomicReference state; + @Nullable + private volatile Exception failure; + + private volatile PersistentTasksService persistentTasksService; + private volatile Logger logger; + private volatile TaskManager taskManager; + + + public AllocatedPersistentTask(long id, String type, String action, String description, TaskId parentTask, + Map headers) { + super(id, type, action, description, parentTask, headers); + this.state = new AtomicReference<>(State.STARTED); + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + + // In case of persistent tasks we always need to return: `false` + // because in case of persistent task the parent task isn't a task in the task manager, but in cluster state. + // This instructs the task manager not to try to kill this persistent task when the task manager cannot find + // a fake parent node id "cluster" in the cluster state + @Override + public final boolean cancelOnParentLeaving() { + return false; + } + + @Override + public Status getStatus() { + return new PersistentTasksNodeService.Status(state.get()); + } + + /** + * Updates the persistent state for the corresponding persistent task. + *

+ * This doesn't affect the status of this allocated task. + */ + public void updatePersistentStatus(Task.Status status, ActionListener> listener) { + persistentTasksService.updateStatus(persistentTaskId, allocationId, status, listener); + } + + public String getPersistentTaskId() { + return persistentTaskId; + } + + void init(PersistentTasksService persistentTasksService, TaskManager taskManager, Logger logger, String persistentTaskId, long + allocationId) { + this.persistentTasksService = persistentTasksService; + this.logger = logger; + this.taskManager = taskManager; + this.persistentTaskId = persistentTaskId; + this.allocationId = allocationId; + } + + public Exception getFailure() { + return failure; + } + + boolean markAsCancelled() { + return state.compareAndSet(AllocatedPersistentTask.State.STARTED, AllocatedPersistentTask.State.PENDING_CANCEL); + } + + public State getState() { + return state.get(); + } + + public long getAllocationId() { + return allocationId; + } + + public enum State { + STARTED, // the task is currently running + PENDING_CANCEL, // the task is cancelled on master, cancelling it locally + COMPLETED // the task is done running and trying to notify caller + } + + /** + * Waits for this persistent task to have the desired state. + */ + public void waitForPersistentTaskStatus(Predicate> predicate, + @Nullable TimeValue timeout, + PersistentTasksService.WaitForPersistentTaskStatusListener listener) { + persistentTasksService.waitForPersistentTaskStatus(persistentTaskId, predicate, timeout, listener); + } + + public void markAsCompleted() { + completeAndNotifyIfNeeded(null); + } + + public void markAsFailed(Exception e) { + if (CancelTasksRequest.DEFAULT_REASON.equals(getReasonCancelled())) { + completeAndNotifyIfNeeded(null); + } else { + completeAndNotifyIfNeeded(e); + } + + } + + private void completeAndNotifyIfNeeded(@Nullable Exception failure) { + State prevState = state.getAndSet(AllocatedPersistentTask.State.COMPLETED); + if (prevState == State.COMPLETED) { + logger.warn("attempt to complete task [{}] with id [{}] in the [{}] state", getAction(), getPersistentTaskId(), prevState); + } else { + if (failure != null) { + logger.warn((Supplier) () -> new ParameterizedMessage( + "task {} failed with an exception", getPersistentTaskId()), failure); + } + try { + this.failure = failure; + if (prevState == State.STARTED) { + logger.trace("sending notification for completed task [{}] with id [{}]", getAction(), getPersistentTaskId()); + persistentTasksService.sendCompletionNotification(getPersistentTaskId(), getAllocationId(), failure, new + ActionListener>() { + @Override + public void onResponse(PersistentTasksCustomMetaData.PersistentTask persistentTask) { + logger.trace("notification for task [{}] with id [{}] was successful", getAction(), + getPersistentTaskId()); + } + + @Override + public void onFailure(Exception e) { + logger.warn((Supplier) () -> + new ParameterizedMessage("notification for task [{}] with id [{}] failed", + getAction(), getPersistentTaskId()), e); + } + }); + } + } finally { + taskManager.unregister(this); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java new file mode 100644 index 0000000000000..c4bffeeb44d22 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java @@ -0,0 +1,191 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Action that is used by executor node to indicate that the persistent action finished or failed on the node and needs to be + * removed from the cluster state in case of successful completion or restarted on some other node in case of failure. + */ +public class CompletionPersistentTaskAction extends Action { + + public static final CompletionPersistentTaskAction INSTANCE = new CompletionPersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/completion"; + + private CompletionPersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + public static class Request extends MasterNodeRequest { + + private String taskId; + + private Exception exception; + + private long allocationId = -1; + + public Request() { + + } + + public Request(String taskId, long allocationId, Exception exception) { + this.taskId = taskId; + this.exception = exception; + this.allocationId = allocationId; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readString(); + allocationId = in.readLong(); + exception = in.readException(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(taskId); + out.writeLong(allocationId); + out.writeException(exception); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (taskId == null) { + validationException = addValidationError("task id is missing", validationException); + } + if (allocationId < 0) { + validationException = addValidationError("allocation id is negative or missing", validationException); + } + return validationException; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(taskId, request.taskId) && + allocationId == request.allocationId && + Objects.equals(exception, request.exception); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, allocationId, exception); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, CompletionPersistentTaskAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTasksClusterService persistentTasksClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTasksClusterService persistentTasksClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTasksClusterService = persistentTasksClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTasksClusterService.completePersistentTask(request.taskId, request.allocationId, request.exception, + new ActionListener>() { + @Override + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java new file mode 100644 index 0000000000000..efed0aef9b807 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/NodePersistentTasksExecutor.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; + +/** + * This component is responsible for execution of persistent tasks. + * + * It abstracts away the execution of tasks and greatly simplifies testing of PersistentTasksNodeService + */ +public class NodePersistentTasksExecutor { + private final ThreadPool threadPool; + + public NodePersistentTasksExecutor(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void executeTask(@Nullable Params params, + @Nullable Task.Status status, + AllocatedPersistentTask task, + PersistentTasksExecutor executor) { + threadPool.executor(executor.getExecutor()).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + task.markAsFailed(e); + } + + @SuppressWarnings("unchecked") + @Override + protected void doRun() throws Exception { + try { + executor.nodeOperation(task, params, status); + } catch (Exception ex) { + task.markAsFailed(ex); + } + + } + }); + + } + +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java new file mode 100644 index 0000000000000..a475a7cde174a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskParams.java @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContentObject; + +/** + * Parameters used to start persistent task + */ +public interface PersistentTaskParams extends NamedWriteable, ToXContentObject { + +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java new file mode 100644 index 0000000000000..4387ea4230fcc --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTaskResponse.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.io.IOException; +import java.util.Objects; + +/** + * Response upon a successful start or an persistent task + */ +public class PersistentTaskResponse extends ActionResponse { + private PersistentTask task; + + public PersistentTaskResponse() { + super(); + } + + public PersistentTaskResponse(PersistentTask task) { + this.task = task; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + task = in.readOptionalWriteable(PersistentTask::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalWriteable(task); + } + + public PersistentTask getTask() { + return task; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTaskResponse that = (PersistentTaskResponse) o; + return Objects.equals(task, that.task); + } + + @Override + public int hashCode() { + return Objects.hash(task); + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java new file mode 100644 index 0000000000000..24d8c5f7be31a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -0,0 +1,329 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.util.Objects; + +/** + * Component that runs only on the master node and is responsible for assigning running tasks to nodes + */ +public class PersistentTasksClusterService extends AbstractComponent implements ClusterStateListener { + + private final ClusterService clusterService; + private final PersistentTasksExecutorRegistry registry; + + public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorRegistry registry, ClusterService clusterService) { + super(settings); + this.clusterService = clusterService; + clusterService.addListener(this); + this.registry = registry; + + } + + /** + * Creates a new persistent task on master node + * + * @param action the action name + * @param params params + * @param listener the listener that will be called when task is started + */ + public void createPersistentTask(String taskId, String action, @Nullable Params params, + ActionListener> listener) { + clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksCustomMetaData.Builder builder = builder(currentState); + if (builder.hasTask(taskId)) { + throw new ResourceAlreadyExistsException("task with id {" + taskId + "} already exist"); + } + validate(action, currentState, params); + final Assignment assignment; + assignment = getAssignement(action, currentState, params); + return update(currentState, builder.addTask(taskId, action, params, assignment)); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @SuppressWarnings("unchecked") + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + PersistentTasksCustomMetaData tasks = newState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + if (tasks != null) { + listener.onResponse(tasks.getTask(taskId)); + } else { + listener.onResponse(null); + } + } + }); + } + + + /** + * Restarts a record about a running persistent task from cluster state + * + * @param id the id of the persistent task + * @param allocationId the allocation id of the persistent task + * @param failure the reason for restarting the task or null if the task completed successfully + * @param listener the listener that will be called when task is removed + */ + public void completePersistentTask(String id, long allocationId, Exception failure, ActionListener> listener) { + final String source; + if (failure != null) { + logger.warn("persistent task " + id + " failed", failure); + source = "finish persistent task (failed)"; + } else { + source = "finish persistent task (success)"; + } + clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); + if (tasksInProgress.hasTask(id, allocationId)) { + tasksInProgress.finishTask(id); + return update(currentState, tasksInProgress); + } else { + if (tasksInProgress.hasTask(id)) { + logger.warn("The task [{}] with id [{}] was found but it has a different allocation id [{}], status is not updated", + PersistentTasksCustomMetaData.getTaskWithId(currentState, id).getTaskName(), id, allocationId); + } else { + logger.warn("The task [{}] wasn't found, status is not updated", id); + } + throw new ResourceNotFoundException("the task with id [" + id + "] and allocation id [" + allocationId + "] not found"); + } + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + // Using old state since in the new state the task is already gone + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(oldState, id)); + } + }); + } + + /** + * Removes the persistent task + * + * @param id the id of a persistent task + * @param listener the listener that will be called when task is removed + */ + public void removePersistentTask(String id, ActionListener> listener) { + clusterService.submitStateUpdateTask("remove persistent task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); + if (tasksInProgress.hasTask(id)) { + return update(currentState, tasksInProgress.removeTask(id)); + } else { + throw new ResourceNotFoundException("the task with id {} doesn't exist", id); + } + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + // Using old state since in the new state the task is already gone + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(oldState, id)); + } + }); + } + + /** + * Update task status + * + * @param id the id of a persistent task + * @param allocationId the expected allocation id of the persistent task + * @param status new status + * @param listener the listener that will be called when task is removed + */ + public void updatePersistentTaskStatus(String id, long allocationId, Task.Status status, ActionListener> listener) { + clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksCustomMetaData.Builder tasksInProgress = builder(currentState); + if (tasksInProgress.hasTask(id, allocationId)) { + return update(currentState, tasksInProgress.updateTaskStatus(id, status)); + } else { + if (tasksInProgress.hasTask(id)) { + logger.warn("trying to update status on task {} with unexpected allocation id {}", id, allocationId); + } else { + logger.warn("trying to update status on non-existing task {}", id); + } + throw new ResourceNotFoundException("the task with id {} and allocation id {} doesn't exist", id, allocationId); + } + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(newState, id)); + } + }); + } + + private Assignment getAssignement(String taskName, ClusterState currentState, + @Nullable Params params) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + return persistentTasksExecutor.getAssignment(params, currentState); + } + + private void validate(String taskName, ClusterState currentState, @Nullable Params params) { + PersistentTasksExecutor persistentTasksExecutor = registry.getPersistentTaskExecutorSafe(taskName); + persistentTasksExecutor.validate(params, currentState); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeMaster()) { + logger.trace("checking task reassignment for cluster state {}", event.state().getVersion()); + if (reassignmentRequired(event, this::getAssignement)) { + logger.trace("task reassignment is needed"); + reassignTasks(); + } else { + logger.trace("task reassignment is not needed"); + } + } + } + + interface ExecutorNodeDecider { + Assignment getAssignment(String action, ClusterState currentState, Params params); + } + + static boolean reassignmentRequired(ClusterChangedEvent event, ExecutorNodeDecider decider) { + PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData prevTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + if (tasks != null && (Objects.equals(tasks, prevTasks) == false || + event.nodesChanged() || + event.routingTableChanged() || + event.previousState().nodes().isLocalNodeElectedMaster() == false)) { + // We need to check if removed nodes were running any of the tasks and reassign them + boolean reassignmentRequired = false; + for (PersistentTask taskInProgress : tasks.tasks()) { + if (taskInProgress.needsReassignment(event.state().nodes())) { + // there is an unassigned task or task with a disappeared node - we need to try assigning it + if (Objects.equals(taskInProgress.getAssignment(), + decider.getAssignment(taskInProgress.getTaskName(), event.state(), taskInProgress.getParams())) == false) { + // it looks like a assignment for at least one task is possible - let's trigger reassignment + reassignmentRequired = true; + break; + } + + } + } + return reassignmentRequired; + } + return false; + } + + /** + * Evaluates the cluster state and tries to assign tasks to nodes + */ + public void reassignTasks() { + clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return reassignTasks(currentState, logger, PersistentTasksClusterService.this::getAssignement); + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn("Unsuccessful persistent task reassignment", e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + + } + }); + } + + static ClusterState reassignTasks(ClusterState currentState, Logger logger, ExecutorNodeDecider decider) { + PersistentTasksCustomMetaData tasks = currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + ClusterState clusterState = currentState; + DiscoveryNodes nodes = currentState.nodes(); + if (tasks != null) { + logger.trace("reassigning {} persistent tasks", tasks.tasks().size()); + // We need to check if removed nodes were running any of the tasks and reassign them + for (PersistentTask task : tasks.tasks()) { + if (task.needsReassignment(nodes)) { + // there is an unassigned task - we need to try assigning it + Assignment assignment = decider.getAssignment(task.getTaskName(), clusterState, task.getParams()); + if (Objects.equals(assignment, task.getAssignment()) == false) { + logger.trace("reassigning task {} from node {} to node {}", task.getId(), + task.getAssignment().getExecutorNode(), assignment.getExecutorNode()); + clusterState = update(clusterState, builder(clusterState).reassignTask(task.getId(), assignment)); + } else { + logger.trace("ignoring task {} because assignment is the same {}", task.getId(), assignment); + } + } else { + logger.trace("ignoring task {} because it is still running", task.getId()); + } + } + } + return clusterState; + } + + private static PersistentTasksCustomMetaData.Builder builder(ClusterState currentState) { + return PersistentTasksCustomMetaData.builder(currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); + } + + private static ClusterState update(ClusterState currentState, PersistentTasksCustomMetaData.Builder tasksInProgress) { + if (tasksInProgress.isChanged()) { + return ClusterState.builder(currentState).metaData( + MetaData.builder(currentState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasksInProgress.build()) + ).build(); + } else { + return currentState; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java new file mode 100644 index 0000000000000..237157e44c43c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java @@ -0,0 +1,692 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.Task.Status; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.elasticsearch.cluster.metadata.MetaData.ALL_CONTEXTS; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * A cluster state record that contains a list of all running persistent tasks + */ +public final class PersistentTasksCustomMetaData extends AbstractNamedDiffable implements MetaData.Custom { + public static final String TYPE = "persistent_tasks"; + + private static final String API_CONTEXT = MetaData.XContentContext.API.toString(); + + // TODO: Implement custom Diff for tasks + private final Map> tasks; + + private final long lastAllocationId; + + public PersistentTasksCustomMetaData(long lastAllocationId, Map> tasks) { + this.lastAllocationId = lastAllocationId; + this.tasks = tasks; + } + + private static final ObjectParser PERSISTENT_TASKS_PARSER = new ObjectParser<>(TYPE, Builder::new); + + private static final ObjectParser, Void> PERSISTENT_TASK_PARSER = + new ObjectParser<>("tasks", TaskBuilder::new); + + public static final ConstructingObjectParser ASSIGNMENT_PARSER = + new ConstructingObjectParser<>("assignment", objects -> new Assignment((String) objects[0], (String) objects[1])); + + private static final NamedObjectParser, Void> TASK_DESCRIPTION_PARSER; + + static { + // Tasks parser initialization + PERSISTENT_TASKS_PARSER.declareLong(Builder::setLastAllocationId, new ParseField("last_allocation_id")); + PERSISTENT_TASKS_PARSER.declareObjectArray(Builder::setTasks, PERSISTENT_TASK_PARSER, new ParseField("tasks")); + + // Task description parser initialization + ObjectParser, String> parser = new ObjectParser<>("named"); + parser.declareObject(TaskDescriptionBuilder::setParams, + (p, c) -> p.namedObject(PersistentTaskParams.class, c, null), new ParseField("params")); + parser.declareObject(TaskDescriptionBuilder::setStatus, + (p, c) -> p.namedObject(Status.class, c, null), new ParseField("status")); + TASK_DESCRIPTION_PARSER = (XContentParser p, Void c, String name) -> parser.parse(p, new TaskDescriptionBuilder<>(name), name); + + // Assignment parser + ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("executor_node")); + ASSIGNMENT_PARSER.declareStringOrNull(constructorArg(), new ParseField("explanation")); + + // Task parser initialization + PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setId, new ParseField("id")); + PERSISTENT_TASK_PARSER.declareString(TaskBuilder::setTaskName, new ParseField("name")); + PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationId, new ParseField("allocation_id")); + + PERSISTENT_TASK_PARSER.declareNamedObjects( + (TaskBuilder taskBuilder, List> objects) -> { + if (objects.size() != 1) { + throw new IllegalArgumentException("only one task description per task is allowed"); + } + TaskDescriptionBuilder builder = objects.get(0); + taskBuilder.setTaskName(builder.taskName); + taskBuilder.setParams(builder.params); + taskBuilder.setStatus(builder.status); + }, TASK_DESCRIPTION_PARSER, new ParseField("task")); + PERSISTENT_TASK_PARSER.declareObject(TaskBuilder::setAssignment, ASSIGNMENT_PARSER, new ParseField("assignment")); + PERSISTENT_TASK_PARSER.declareLong(TaskBuilder::setAllocationIdOnLastStatusUpdate, + new ParseField("allocation_id_on_last_status_update")); + } + + /** + * Private builder used in XContent parser to build task-specific portion (params and status) + */ + private static class TaskDescriptionBuilder { + private final String taskName; + private Params params; + private Status status; + + private TaskDescriptionBuilder(String taskName) { + this.taskName = taskName; + } + + private TaskDescriptionBuilder setParams(Params params) { + this.params = params; + return this; + } + + private TaskDescriptionBuilder setStatus(Status status) { + this.status = status; + return this; + } + } + + + public Collection> tasks() { + return this.tasks.values(); + } + + public Map> taskMap() { + return this.tasks; + } + + public PersistentTask getTask(String id) { + return this.tasks.get(id); + } + + public Collection> findTasks(String taskName, Predicate> predicate) { + return this.tasks().stream() + .filter(p -> taskName.equals(p.getTaskName())) + .filter(predicate) + .collect(Collectors.toList()); + } + + public boolean tasksExist(String taskName, Predicate> predicate) { + return this.tasks().stream() + .filter(p -> taskName.equals(p.getTaskName())) + .anyMatch(predicate); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTasksCustomMetaData that = (PersistentTasksCustomMetaData) o; + return lastAllocationId == that.lastAllocationId && + Objects.equals(tasks, that.tasks); + } + + @Override + public int hashCode() { + return Objects.hash(tasks, lastAllocationId); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + public long getNumberOfTasksOnNode(String nodeId, String taskName) { + return tasks.values().stream().filter( + task -> taskName.equals(task.taskName) && nodeId.equals(task.assignment.executorNode)).count(); + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_4_0; + } + + @Override + public EnumSet context() { + return ALL_CONTEXTS; + } + + public static PersistentTasksCustomMetaData fromXContent(XContentParser parser) throws IOException { + return PERSISTENT_TASKS_PARSER.parse(parser, null).build(); + } + + @SuppressWarnings("unchecked") + public static PersistentTask getTaskWithId(ClusterState clusterState, String taskId) { + PersistentTasksCustomMetaData tasks = clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE); + if (tasks != null) { + return (PersistentTask) tasks.getTask(taskId); + } + return null; + } + + public static class Assignment { + @Nullable + private final String executorNode; + private final String explanation; + + public Assignment(String executorNode, String explanation) { + this.executorNode = executorNode; + assert explanation != null; + this.explanation = explanation; + } + + @Nullable + public String getExecutorNode() { + return executorNode; + } + + public String getExplanation() { + return explanation; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Assignment that = (Assignment) o; + return Objects.equals(executorNode, that.executorNode) && + Objects.equals(explanation, that.explanation); + } + + @Override + public int hashCode() { + return Objects.hash(executorNode, explanation); + } + + public boolean isAssigned() { + return executorNode != null; + } + + @Override + public String toString() { + return "node: [" + executorNode + "], explanation: [" + explanation + "]"; + } + } + + public static final Assignment INITIAL_ASSIGNMENT = new Assignment(null, "waiting for initial assignment"); + + /** + * A record that represents a single running persistent task + */ + public static class PersistentTask

implements Writeable, ToXContentObject { + private final String id; + private final long allocationId; + private final String taskName; + @Nullable + private final P params; + @Nullable + private final Status status; + private final Assignment assignment; + @Nullable + private final Long allocationIdOnLastStatusUpdate; + + + public PersistentTask(String id, String taskName, P params, long allocationId, Assignment assignment) { + this(id, allocationId, taskName, params, null, assignment, null); + } + + public PersistentTask(PersistentTask

task, long allocationId, Assignment assignment) { + this(task.id, allocationId, task.taskName, task.params, task.status, + assignment, task.allocationId); + } + + public PersistentTask(PersistentTask

task, Status status) { + this(task.id, task.allocationId, task.taskName, task.params, status, + task.assignment, task.allocationId); + } + + private PersistentTask(String id, long allocationId, String taskName, P params, + Status status, Assignment assignment, Long allocationIdOnLastStatusUpdate) { + this.id = id; + this.allocationId = allocationId; + this.taskName = taskName; + this.params = params; + this.status = status; + this.assignment = assignment; + this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; + if (params != null) { + if (params.getWriteableName().equals(taskName) == false) { + throw new IllegalArgumentException("params have to have the same writeable name as task. params: " + + params.getWriteableName() + " task: " + taskName); + } + } + if (status != null) { + if (status.getWriteableName().equals(taskName) == false) { + throw new IllegalArgumentException("status has to have the same writeable name as task. status: " + + status.getWriteableName() + " task: " + taskName); + } + } + } + + @SuppressWarnings("unchecked") + public PersistentTask(StreamInput in) throws IOException { + id = in.readString(); + allocationId = in.readLong(); + taskName = in.readString(); + params = (P) in.readOptionalNamedWriteable(PersistentTaskParams.class); + status = in.readOptionalNamedWriteable(Task.Status.class); + assignment = new Assignment(in.readOptionalString(), in.readString()); + allocationIdOnLastStatusUpdate = in.readOptionalLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(allocationId); + out.writeString(taskName); + out.writeOptionalNamedWriteable(params); + out.writeOptionalNamedWriteable(status); + out.writeOptionalString(assignment.executorNode); + out.writeString(assignment.explanation); + out.writeOptionalLong(allocationIdOnLastStatusUpdate); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTask that = (PersistentTask) o; + return Objects.equals(id, that.id) && + allocationId == that.allocationId && + Objects.equals(taskName, that.taskName) && + Objects.equals(params, that.params) && + Objects.equals(status, that.status) && + Objects.equals(assignment, that.assignment) && + Objects.equals(allocationIdOnLastStatusUpdate, that.allocationIdOnLastStatusUpdate); + } + + @Override + public int hashCode() { + return Objects.hash(id, allocationId, taskName, params, status, assignment, + allocationIdOnLastStatusUpdate); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + public String getId() { + return id; + } + + public long getAllocationId() { + return allocationId; + } + + public String getTaskName() { + return taskName; + } + + @Nullable + public P getParams() { + return params; + } + + @Nullable + public String getExecutorNode() { + return assignment.executorNode; + } + + public Assignment getAssignment() { + return assignment; + } + + public boolean isAssigned() { + return assignment.isAssigned(); + } + + /** + * Returns true if the tasks is not stopped and unassigned or assigned to a non-existing node. + */ + public boolean needsReassignment(DiscoveryNodes nodes) { + return (assignment.isAssigned() == false || nodes.nodeExists(assignment.getExecutorNode()) == false); + } + + @Nullable + public Status getStatus() { + return status; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params xParams) throws IOException { + builder.startObject(); + { + builder.field("id", id); + builder.startObject("task"); + { + builder.startObject(taskName); + { + if (params != null) { + builder.field("params", params, xParams); + } + if (status != null) { + builder.field("status", status, xParams); + } + } + builder.endObject(); + } + builder.endObject(); + + if (API_CONTEXT.equals(xParams.param(MetaData.CONTEXT_MODE_PARAM, API_CONTEXT))) { + // These are transient values that shouldn't be persisted to gateway cluster state or snapshot + builder.field("allocation_id", allocationId); + builder.startObject("assignment"); + { + builder.field("executor_node", assignment.executorNode); + builder.field("explanation", assignment.explanation); + } + builder.endObject(); + if (allocationIdOnLastStatusUpdate != null) { + builder.field("allocation_id_on_last_status_update", allocationIdOnLastStatusUpdate); + } + } + } + builder.endObject(); + return builder; + } + + @Override + public boolean isFragment() { + return false; + } + } + + private static class TaskBuilder { + private String id; + private long allocationId; + private String taskName; + private Params params; + private Status status; + private Assignment assignment = INITIAL_ASSIGNMENT; + private Long allocationIdOnLastStatusUpdate; + + public TaskBuilder setId(String id) { + this.id = id; + return this; + } + + public TaskBuilder setAllocationId(long allocationId) { + this.allocationId = allocationId; + return this; + } + + public TaskBuilder setTaskName(String taskName) { + this.taskName = taskName; + return this; + } + + public TaskBuilder setParams(Params params) { + this.params = params; + return this; + } + + public TaskBuilder setStatus(Status status) { + this.status = status; + return this; + } + + + public TaskBuilder setAssignment(Assignment assignment) { + this.assignment = assignment; + return this; + } + + public TaskBuilder setAllocationIdOnLastStatusUpdate(Long allocationIdOnLastStatusUpdate) { + this.allocationIdOnLastStatusUpdate = allocationIdOnLastStatusUpdate; + return this; + } + + public PersistentTask build() { + return new PersistentTask<>(id, allocationId, taskName, params, status, + assignment, allocationIdOnLastStatusUpdate); + } + } + + @Override + public String getWriteableName() { + return TYPE; + } + + public PersistentTasksCustomMetaData(StreamInput in) throws IOException { + lastAllocationId = in.readLong(); + tasks = in.readMap(StreamInput::readString, PersistentTask::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(lastAllocationId); + out.writeMap(tasks, StreamOutput::writeString, (stream, value) -> value.writeTo(stream)); + } + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(MetaData.Custom.class, TYPE, in); + } + + public long getLastAllocationId() { + return lastAllocationId; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("last_allocation_id", lastAllocationId); + builder.startArray("tasks"); + for (PersistentTask entry : tasks.values()) { + entry.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(PersistentTasksCustomMetaData tasks) { + return new Builder(tasks); + } + + public static class Builder { + private final Map> tasks = new HashMap<>(); + private long lastAllocationId; + private boolean changed; + + private Builder() { + } + + private Builder(PersistentTasksCustomMetaData tasksInProgress) { + if (tasksInProgress != null) { + tasks.putAll(tasksInProgress.tasks); + lastAllocationId = tasksInProgress.lastAllocationId; + } else { + lastAllocationId = 0; + } + } + + public long getLastAllocationId() { + return lastAllocationId; + } + + private Builder setLastAllocationId(long currentId) { + this.lastAllocationId = currentId; + return this; + } + + private Builder setTasks(List> tasks) { + for (TaskBuilder builder : tasks) { + PersistentTask task = builder.build(); + this.tasks.put(task.getId(), task); + } + return this; + } + + private long getNextAllocationId() { + lastAllocationId++; + return lastAllocationId; + } + + /** + * Adds a new task to the builder + *

+ * After the task is added its id can be found by calling {{@link #getLastAllocationId()}} method. + */ + public Builder addTask(String taskId, String taskName, Params params, + Assignment assignment) { + changed = true; + PersistentTask previousTask = tasks.put(taskId, new PersistentTask<>(taskId, taskName, params, + getNextAllocationId(), assignment)); + if (previousTask != null) { + throw new ResourceAlreadyExistsException("Trying to override task with id {" + taskId + "}"); + } + return this; + } + + /** + * Reassigns the task to another node + */ + public Builder reassignTask(String taskId, Assignment assignment) { + PersistentTask taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + changed = true; + tasks.put(taskId, new PersistentTask<>(taskInProgress, getNextAllocationId(), assignment)); + } else { + throw new ResourceNotFoundException("cannot reassign task with id {" + taskId + "}, the task no longer exits"); + } + return this; + } + + /** + * Updates the task status + */ + public Builder updateTaskStatus(String taskId, Status status) { + PersistentTask taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + changed = true; + tasks.put(taskId, new PersistentTask<>(taskInProgress, status)); + } else { + throw new ResourceNotFoundException("cannot update task with id {" + taskId + "}, the task no longer exits"); + } + return this; + } + + /** + * Removes the task + */ + public Builder removeTask(String taskId) { + if (tasks.remove(taskId) != null) { + changed = true; + } else { + throw new ResourceNotFoundException("cannot remove task with id {" + taskId + "}, the task no longer exits"); + } + return this; + } + + /** + * Finishes the task + *

+ * If the task is marked with removeOnCompletion flag, it is removed from the list, otherwise it is stopped. + */ + public Builder finishTask(String taskId) { + PersistentTask taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + changed = true; + tasks.remove(taskId); + } else { + throw new ResourceNotFoundException("cannot finish task with id {" + taskId + "}, the task no longer exits"); + } + return this; + } + + /** + * Checks if the task is currently present in the list + */ + public boolean hasTask(String taskId) { + return tasks.containsKey(taskId); + } + + /** + * Checks if the task is currently present in the list and has the right allocation id + */ + public boolean hasTask(String taskId, long allocationId) { + PersistentTask taskInProgress = tasks.get(taskId); + if (taskInProgress != null) { + return taskInProgress.getAllocationId() == allocationId; + } + return false; + } + + Set getCurrentTaskIds() { + return tasks.keySet(); + } + + /** + * Returns true if any the task list was changed since the builder was created + */ + public boolean isChanged() { + return changed; + } + + public PersistentTasksCustomMetaData build() { + return new PersistentTasksCustomMetaData(lastAllocationId, Collections.unmodifiableMap(tasks)); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java new file mode 100644 index 0000000000000..ed61ad5805391 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.util.Map; +import java.util.function.Predicate; + +/** + * An executor of tasks that can survive restart of requesting or executing node. + * These tasks are using cluster state rather than only transport service to send requests and responses. + */ +public abstract class PersistentTasksExecutor extends AbstractComponent { + + private final String executor; + private final String taskName; + + protected PersistentTasksExecutor(Settings settings, String taskName, String executor) { + super(settings); + this.taskName = taskName; + this.executor = executor; + } + + public String getTaskName() { + return taskName; + } + + public static final Assignment NO_NODE_FOUND = new Assignment(null, "no appropriate nodes found for the assignment"); + + /** + * Returns the node id where the params has to be executed, + *

+ * The default implementation returns the least loaded data node + */ + public Assignment getAssignment(Params params, ClusterState clusterState) { + DiscoveryNode discoveryNode = selectLeastLoadedNode(clusterState, DiscoveryNode::isDataNode); + if (discoveryNode == null) { + return NO_NODE_FOUND; + } else { + return new Assignment(discoveryNode.getId(), ""); + } + } + + /** + * Finds the least loaded node that satisfies the selector criteria + */ + protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate selector) { + long minLoad = Long.MAX_VALUE; + DiscoveryNode minLoadedNode = null; + PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + for (DiscoveryNode node : clusterState.getNodes()) { + if (selector.test(node)) { + if (persistentTasks == null) { + // We don't have any task running yet, pick the first available node + return node; + } + long numberOfTasks = persistentTasks.getNumberOfTasksOnNode(node.getId(), taskName); + if (minLoad > numberOfTasks) { + minLoad = numberOfTasks; + minLoadedNode = node; + } + } + } + return minLoadedNode; + } + + /** + * Checks the current cluster state for compatibility with the params + *

+ * Throws an exception if the supplied params cannot be executed on the cluster in the current state. + */ + public void validate(Params params, ClusterState clusterState) { + + } + + /** + * Creates a AllocatedPersistentTask for communicating with task manager + */ + protected AllocatedPersistentTask createTask(long id, String type, String action, TaskId parentTaskId, + PersistentTask taskInProgress, Map headers) { + return new AllocatedPersistentTask(id, type, action, getDescription(taskInProgress), parentTaskId, headers); + } + + /** + * Returns task description that will be available via task manager + */ + protected String getDescription(PersistentTask taskInProgress) { + return "id=" + taskInProgress.getId(); + } + + /** + * This operation will be executed on the executor node. + *

+ * NOTE: The nodeOperation has to throw an exception, trigger task.markAsCompleted() or task.completeAndNotifyIfNeeded() methods to + * indicate that the persistent task has finished. + */ + protected abstract void nodeOperation(AllocatedPersistentTask task, @Nullable Params params, @Nullable Task.Status status); + + public String getExecutor() { + return executor; + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java new file mode 100644 index 0000000000000..2ac57e074b7bf --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Components that registers all persistent task executors + */ +public class PersistentTasksExecutorRegistry extends AbstractComponent { + + private final Map> taskExecutors; + + @SuppressWarnings("unchecked") + public PersistentTasksExecutorRegistry(Settings settings, Collection> taskExecutors) { + super(settings); + Map> map = new HashMap<>(); + for (PersistentTasksExecutor executor : taskExecutors) { + map.put(executor.getTaskName(), executor); + } + this.taskExecutors = Collections.unmodifiableMap(map); + } + + @SuppressWarnings("unchecked") + public PersistentTasksExecutor getPersistentTaskExecutorSafe(String taskName) { + PersistentTasksExecutor executor = (PersistentTasksExecutor) taskExecutors.get(taskName); + if (executor == null) { + throw new IllegalStateException("Unknown persistent executor [" + taskName + "]"); + } + return executor; + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java new file mode 100644 index 0000000000000..e53834d6f4655 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java @@ -0,0 +1,278 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskAwareRequest; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +/** + * This component is responsible for coordination of execution of persistent tasks on individual nodes. It runs on all + * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. + */ +public class PersistentTasksNodeService extends AbstractComponent implements ClusterStateListener { + private final Map runningTasks = new HashMap<>(); + private final PersistentTasksService persistentTasksService; + private final PersistentTasksExecutorRegistry persistentTasksExecutorRegistry; + private final TaskManager taskManager; + private final NodePersistentTasksExecutor nodePersistentTasksExecutor; + + + public PersistentTasksNodeService(Settings settings, + PersistentTasksService persistentTasksService, + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, + TaskManager taskManager, NodePersistentTasksExecutor nodePersistentTasksExecutor) { + super(settings); + this.persistentTasksService = persistentTasksService; + this.persistentTasksExecutorRegistry = persistentTasksExecutorRegistry; + this.taskManager = taskManager; + this.nodePersistentTasksExecutor = nodePersistentTasksExecutor; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + // wait until the gateway has recovered from disk, otherwise if the only master restarts + // we start cancelling all local tasks before cluster has a chance to recover. + return; + } + PersistentTasksCustomMetaData tasks = event.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + PersistentTasksCustomMetaData previousTasks = event.previousState().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + + // Cluster State Local State Local Action + // STARTED NULL Create as STARTED, Start + // STARTED STARTED Noop - running + // STARTED COMPLETED Noop - waiting for notification ack + + // NULL NULL Noop - nothing to do + // NULL STARTED Remove locally, Mark as PENDING_CANCEL, Cancel + // NULL COMPLETED Remove locally + + // Master states: + // NULL - doesn't exist in the cluster state + // STARTED - exist in the cluster state + + // Local state: + // NULL - we don't have task registered locally in runningTasks + // STARTED - registered in TaskManager, requires master notification when finishes + // PENDING_CANCEL - registered in TaskManager, doesn't require master notification when finishes + // COMPLETED - not registered in TaskManager, notified, waiting for master to remove it from CS so we can remove locally + + // When task finishes if it is marked as STARTED or PENDING_CANCEL it is marked as COMPLETED and unregistered, + // If the task was STARTED, the master notification is also triggered (this is handled by unregisterTask() method, which is + // triggered by PersistentTaskListener + + if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { + // We have some changes let's check if they are related to our node + String localNodeId = event.state().getNodes().getLocalNodeId(); + Set notVisitedTasks = new HashSet<>(runningTasks.keySet()); + if (tasks != null) { + for (PersistentTask taskInProgress : tasks.tasks()) { + if (localNodeId.equals(taskInProgress.getExecutorNode())) { + Long allocationId = taskInProgress.getAllocationId(); + AllocatedPersistentTask persistentTask = runningTasks.get(allocationId); + if (persistentTask == null) { + // New task - let's start it + startTask(taskInProgress); + } else { + // The task is still running + notVisitedTasks.remove(allocationId); + } + } + } + } + + for (Long id : notVisitedTasks) { + AllocatedPersistentTask task = runningTasks.get(id); + if (task.getState() == AllocatedPersistentTask.State.COMPLETED) { + // Result was sent to the caller and the caller acknowledged acceptance of the result + logger.trace("Found completed persistent task [{}] with id [{}] and allocation id [{}] - removing", + task.getAction(), task.getPersistentTaskId(), task.getAllocationId()); + runningTasks.remove(id); + } else { + // task is running locally, but master doesn't know about it - that means that the persistent task was removed + // cancel the task without notifying master + logger.trace("Found unregistered persistent task [{}] with id [{}] and allocation id [{}] - cancelling", + task.getAction(), task.getPersistentTaskId(), task.getAllocationId()); + cancelTask(id); + } + } + + } + + } + + private void startTask(PersistentTask taskInProgress) { + PersistentTasksExecutor executor = + persistentTasksExecutorRegistry.getPersistentTaskExecutorSafe(taskInProgress.getTaskName()); + + TaskAwareRequest request = new TaskAwareRequest() { + TaskId parentTaskId = new TaskId("cluster", taskInProgress.getAllocationId()); + + @Override + public void setParentTask(TaskId taskId) { + throw new UnsupportedOperationException("parent task if for persistent tasks shouldn't change"); + } + + @Override + public TaskId getParentTask() { + return parentTaskId; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return executor.createTask(id, type, action, parentTaskId, taskInProgress, headers); + } + }; + AllocatedPersistentTask task = (AllocatedPersistentTask) taskManager.register("persistent", taskInProgress.getTaskName() + "[c]", + request); + boolean processed = false; + try { + task.init(persistentTasksService, taskManager, logger, taskInProgress.getId(), taskInProgress.getAllocationId()); + logger.trace("Persistent task [{}] with id [{}] and allocation id [{}] was created", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()); + try { + runningTasks.put(taskInProgress.getAllocationId(), task); + nodePersistentTasksExecutor.executeTask(taskInProgress.getParams(), taskInProgress.getStatus(), task, executor); + } catch (Exception e) { + // Submit task failure + task.markAsFailed(e); + } + processed = true; + } finally { + if (processed == false) { + // something went wrong - unregistering task + logger.warn("Persistent task [{}] with id [{}] and allocation id [{}] failed to create", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()); + taskManager.unregister(task); + } + } + } + + /** + * Unregisters and then cancels the locally running task using the task manager. No notification to master will be send upon + * cancellation. + */ + private void cancelTask(Long allocationId) { + AllocatedPersistentTask task = runningTasks.remove(allocationId); + if (task.markAsCancelled()) { + // Cancel the local task using the task manager + persistentTasksService.sendTaskManagerCancellation(task.getId(), new ActionListener() { + @Override + public void onResponse(CancelTasksResponse cancelTasksResponse) { + logger.trace("Persistent task [{}] with id [{}] and allocation id [{}] was cancelled", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()); + } + + @Override + public void onFailure(Exception e) { + // There is really nothing we can do in case of failure here + logger.warn((Supplier) () -> + new ParameterizedMessage("failed to cancel task [{}] with id [{}] and allocation id [{}]", task.getAction(), + task.getPersistentTaskId(), task.getAllocationId()), e); + } + }); + } + } + + + public static class Status implements Task.Status { + public static final String NAME = "persistent_executor"; + + private final AllocatedPersistentTask.State state; + + public Status(AllocatedPersistentTask.State state) { + this.state = requireNonNull(state, "State cannot be null"); + } + + public Status(StreamInput in) throws IOException { + state = AllocatedPersistentTask.State.valueOf(in.readString()); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("state", state.toString()); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(state.toString()); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + public AllocatedPersistentTask.State getState() { + return state; + } + + @Override + public boolean isFragment() { + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return state == status.state; + } + + @Override + public int hashCode() { + return Objects.hash(state); + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java new file mode 100644 index 0000000000000..2b656dd219cf9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java @@ -0,0 +1,238 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.action.support.ContextPreservingActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.node.NodeClosedException; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.util.function.BiConsumer; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** + * This service is used by persistent actions to propagate changes in the action state and notify about completion + */ +public class PersistentTasksService extends AbstractComponent { + + private final Client client; + private final ClusterService clusterService; + private final ThreadPool threadPool; + + public PersistentTasksService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) { + super(settings); + this.client = client; + this.clusterService = clusterService; + this.threadPool = threadPool; + } + + /** + * Creates the specified persistent task and attempts to assign it to a node. + */ + @SuppressWarnings("unchecked") + public void startPersistentTask(String taskId, String taskName, @Nullable Params params, + ActionListener> listener) { + StartPersistentTaskAction.Request createPersistentActionRequest = + new StartPersistentTaskAction.Request(taskId, taskName, params); + try { + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, StartPersistentTaskAction.INSTANCE, createPersistentActionRequest, + ActionListener.wrap(o -> listener.onResponse((PersistentTask) o.getTask()), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Notifies the PersistentTasksClusterService about successful (failure == null) completion of a task or its failure + */ + public void sendCompletionNotification(String taskId, long allocationId, Exception failure, + ActionListener> listener) { + CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, allocationId, failure); + try { + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, CompletionPersistentTaskAction.INSTANCE, restartRequest, + ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Cancels a locally running task using the task manager + */ + void sendTaskManagerCancellation(long taskId, ActionListener listener) { + DiscoveryNode localNode = clusterService.localNode(); + CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); + cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); + cancelTasksRequest.setReason("persistent action was removed"); + try { + executeAsyncWithOrigin(client.threadPool().getThreadContext(), PERSISTENT_TASK_ORIGIN, cancelTasksRequest, listener, + client.admin().cluster()::cancelTasks); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Updates status of the persistent task. + *

+ * Persistent task implementers shouldn't call this method directly and use + * {@link AllocatedPersistentTask#updatePersistentStatus} instead + */ + void updateStatus(String taskId, long allocationId, Task.Status status, ActionListener> listener) { + UpdatePersistentTaskStatusAction.Request updateStatusRequest = + new UpdatePersistentTaskStatusAction.Request(taskId, allocationId, status); + try { + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, + ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Cancels if needed and removes a persistent task + */ + public void cancelPersistentTask(String taskId, ActionListener> listener) { + RemovePersistentTaskAction.Request removeRequest = new RemovePersistentTaskAction.Request(taskId); + try { + executeAsyncWithOrigin(client, PERSISTENT_TASK_ORIGIN, RemovePersistentTaskAction.INSTANCE, removeRequest, + ActionListener.wrap(o -> listener.onResponse(o.getTask()), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Checks if the persistent task with giving id (taskId) has the desired state and if it doesn't + * waits of it. + */ + public void waitForPersistentTaskStatus(String taskId, Predicate> predicate, @Nullable TimeValue timeout, + WaitForPersistentTaskStatusListener listener) { + ClusterStateObserver stateObserver = new ClusterStateObserver(clusterService, timeout, logger, threadPool.getThreadContext()); + if (predicate.test(PersistentTasksCustomMetaData.getTaskWithId(stateObserver.setAndGetObservedState(), taskId))) { + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(stateObserver.setAndGetObservedState(), taskId)); + } else { + stateObserver.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + listener.onResponse(PersistentTasksCustomMetaData.getTaskWithId(state, taskId)); + } + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + listener.onTimeout(timeout); + } + }, clusterState -> predicate.test(PersistentTasksCustomMetaData.getTaskWithId(clusterState, taskId))); + } + } + + public void waitForPersistentTasksStatus(Predicate predicate, + @Nullable TimeValue timeout, ActionListener listener) { + ClusterStateObserver stateObserver = new ClusterStateObserver(clusterService, timeout, + logger, threadPool.getThreadContext()); + if (predicate.test(stateObserver.setAndGetObservedState().metaData().custom(PersistentTasksCustomMetaData.TYPE))) { + listener.onResponse(true); + } else { + stateObserver.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + listener.onResponse(true); + } + + @Override + public void onClusterServiceClose() { + listener.onFailure(new NodeClosedException(clusterService.localNode())); + } + + @Override + public void onTimeout(TimeValue timeout) { + listener.onFailure(new IllegalStateException("timed out after " + timeout)); + } + }, clusterState -> predicate.test(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)), timeout); + } + } + + public interface WaitForPersistentTaskStatusListener + extends ActionListener> { + default void onTimeout(TimeValue timeout) { + onFailure(new IllegalStateException("timed out after " + timeout)); + } + } + + private static final String ACTION_ORIGIN_TRANSIENT_NAME = "action.origin"; + private static final String PERSISTENT_TASK_ORIGIN = "persistent_tasks"; + + /** + * Executes a consumer after setting the origin and wrapping the listener so that the proper context is restored + */ + public static void executeAsyncWithOrigin( + ThreadContext threadContext, String origin, Request request, ActionListener listener, + BiConsumer> consumer) { + final Supplier supplier = threadContext.newRestorableContext(false); + try (ThreadContext.StoredContext ignore = stashWithOrigin(threadContext, origin)) { + consumer.accept(request, new ContextPreservingActionListener<>(supplier, listener)); + } + } + /** + * Executes an asynchronous action using the provided client. The origin is set in the context and the listener + * is wrapped to ensure the proper context is restored + */ + public static > void executeAsyncWithOrigin( + Client client, String origin, Action action, Request request, + ActionListener listener) { + final ThreadContext threadContext = client.threadPool().getThreadContext(); + final Supplier supplier = threadContext.newRestorableContext(false); + try (ThreadContext.StoredContext ignore = stashWithOrigin(threadContext, origin)) { + client.execute(action, request, new ContextPreservingActionListener<>(supplier, listener)); + } + } + + public static ThreadContext.StoredContext stashWithOrigin(ThreadContext threadContext, String origin) { + final ThreadContext.StoredContext storedContext = threadContext.stashContext(); + threadContext.putTransient(ACTION_ORIGIN_TRANSIENT_NAME, origin); + return storedContext; + } + +} diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java new file mode 100644 index 0000000000000..d0a4575a10303 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java @@ -0,0 +1,175 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.io.IOException; +import java.util.Objects; + +public class RemovePersistentTaskAction extends Action { + + public static final RemovePersistentTaskAction INSTANCE = new RemovePersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/remove"; + + private RemovePersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + public static class Request extends MasterNodeRequest { + + private String taskId; + + public Request() { + + } + + public Request(String taskId) { + this.taskId = taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(taskId); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(taskId, request.taskId); + } + + @Override + public int hashCode() { + return Objects.hash(taskId); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, RemovePersistentTaskAction action) { + super(client, action, new Request()); + } + + public final RequestBuilder setTaskId(String taskId) { + request.setTaskId(taskId); + return this; + } + + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTasksClusterService persistentTasksClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTasksClusterService persistentTasksClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTasksClusterService = persistentTasksClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTasksClusterService.removePersistentTask(request.taskId, new ActionListener>() { + @Override + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java new file mode 100644 index 0000000000000..3b988939879c5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java @@ -0,0 +1,245 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * This action can be used to add the record for the persistent action to the cluster state. + */ +public class StartPersistentTaskAction extends Action { + + public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/start"; + + private StartPersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + public static class Request extends MasterNodeRequest { + + private String taskId; + + @Nullable + private String taskName; + + private PersistentTaskParams params; + + public Request() { + + } + + public Request(String taskId, String taskName, PersistentTaskParams params) { + this.taskId = taskId; + this.taskName = taskName; + this.params = params; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readString(); + taskName = in.readString(); + params = in.readOptionalNamedWriteable(PersistentTaskParams.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(taskId); + out.writeString(taskName); + out.writeOptionalNamedWriteable(params); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (this.taskId == null) { + validationException = addValidationError("task id must be specified", validationException); + } + if (this.taskName == null) { + validationException = addValidationError("action must be specified", validationException); + } + if (params != null) { + if (params.getWriteableName().equals(taskName) == false) { + validationException = addValidationError("params have to have the same writeable name as task. params: " + + params.getWriteableName() + " task: " + taskName, validationException); + } + } + return validationException; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request1 = (Request) o; + return Objects.equals(taskId, request1.taskId) && Objects.equals(taskName, request1.taskName) && + Objects.equals(params, request1.params); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, taskName, params); + } + + public String getTaskName() { + return taskName; + } + + public void setTaskName(String taskName) { + this.taskName = taskName; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public PersistentTaskParams getParams() { + return params; + } + + @Nullable + public void setParams(PersistentTaskParams params) { + this.params = params; + } + + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, StartPersistentTaskAction action) { + super(client, action, new Request()); + } + + public RequestBuilder setTaskId(String taskId) { + request.setTaskId(taskId); + return this; + } + + public RequestBuilder setAction(String action) { + request.setTaskName(action); + return this; + } + + public RequestBuilder setRequest(PersistentTaskParams params) { + request.setParams(params); + return this; + } + + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTasksClusterService persistentTasksClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTasksClusterService persistentTasksClusterService, + PersistentTasksExecutorRegistry persistentTasksExecutorRegistry, + PersistentTasksService persistentTasksService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTasksClusterService = persistentTasksClusterService; + NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool); + clusterService.addListener(new PersistentTasksNodeService(settings, persistentTasksService, persistentTasksExecutorRegistry, + transportService.getTaskManager(), executor)); + } + + @Override + protected String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTasksClusterService.createPersistentTask(request.taskId, request.taskName, request.params, + new ActionListener>() { + + @Override + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java new file mode 100644 index 0000000000000..53bc9afd0fdf5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java @@ -0,0 +1,208 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class UpdatePersistentTaskStatusAction extends Action { + + public static final UpdatePersistentTaskStatusAction INSTANCE = new UpdatePersistentTaskStatusAction(); + public static final String NAME = "cluster:admin/persistent/update_status"; + + private UpdatePersistentTaskStatusAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + public static class Request extends MasterNodeRequest { + + private String taskId; + private long allocationId = -1L; + private Task.Status status; + + public Request() { + + } + + public Request(String taskId, long allocationId, Task.Status status) { + this.taskId = taskId; + this.allocationId = allocationId; + this.status = status; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public void setAllocationId(long allocationId) { + this.allocationId = allocationId; + } + + public void setStatus(Task.Status status) { + this.status = status; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readString(); + allocationId = in.readLong(); + status = in.readOptionalNamedWriteable(Task.Status.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(taskId); + out.writeLong(allocationId); + out.writeOptionalNamedWriteable(status); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (this.taskId == null) { + validationException = addValidationError("task id must be specified", validationException); + } + if (this.allocationId == -1L) { + validationException = addValidationError("allocationId must be specified", validationException); + } + // We cannot really check if status has the same type as task because we don't have access + // to the task here. We will check it when we try to update the task + return validationException; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(taskId, request.taskId) && allocationId == request.allocationId && + Objects.equals(status, request.status); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, allocationId, status); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, UpdatePersistentTaskStatusAction action) { + super(client, action, new Request()); + } + + public final RequestBuilder setTaskId(String taskId) { + request.setTaskId(taskId); + return this; + } + + public final RequestBuilder setStatus(Task.Status status) { + request.setStatus(status); + return this; + } + + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTasksClusterService persistentTasksClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTasksClusterService persistentTasksClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTasksClusterService = persistentTasksClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected PersistentTaskResponse newResponse() { + return new PersistentTaskResponse(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTasksClusterService.updatePersistentTaskStatus(request.taskId, request.allocationId, request.status, + new ActionListener>() { + @Override + public void onResponse(PersistentTask task) { + listener.onResponse(new PersistentTaskResponse(task)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/persistent/package-info.java b/server/src/main/java/org/elasticsearch/persistent/package-info.java new file mode 100644 index 0000000000000..f948e3ace448e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/persistent/package-info.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The Persistent Tasks Executors are responsible for executing restartable tasks that can survive disappearance of a + * coordinating and executor nodes. + *

+ * In order to be resilient to node restarts, the persistent tasks are using the cluster state instead of a transport service to send + * requests and responses. The execution is done in six phases: + *

+ * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent task. This task is handled + * by the {@link org.elasticsearch.persistent.PersistentTasksService}, which is using + * {@link org.elasticsearch.persistent.PersistentTasksClusterService} to update cluster state with the record about running persistent + * task. + *

+ * 2. The master node updates the {@link org.elasticsearch.persistent.PersistentTasksCustomMetaData} in the cluster state to indicate + * that there is a new persistent task is running in the system. + *

+ * 3. The {@link org.elasticsearch.persistent.PersistentTasksNodeService} running on every node in the cluster monitors changes in + * the cluster state and starts execution of all new tasks assigned to the node it is running on. + *

+ * 4. If the task fails to start on the node, the {@link org.elasticsearch.persistent.PersistentTasksNodeService} uses the + * {@link org.elasticsearch.persistent.PersistentTasksCustomMetaData} to notify the + * {@link org.elasticsearch.persistent.PersistentTasksService}, which reassigns the action to another node in the cluster. + *

+ * 5. If a task finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the + * cluster state unless removeOnCompletion flag for this task is set to false. + *

+ * 6. The {@link org.elasticsearch.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent task. + */ +package org.elasticsearch.persistent; diff --git a/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java b/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java new file mode 100644 index 0000000000000..c402b907ffde6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.plugins; + +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.persistent.PersistentTasksExecutor; + +import java.util.Collections; +import java.util.List; + +/** + * Plugin for registering persistent tasks executors. + */ +public interface PersistentTaskPlugin { + + /** + * Returns additional persistent tasks executors added by this plugin. + */ + default List> getPersistentTasksExecutor(ClusterService clusterService) { + return Collections.emptyList(); + } + +} diff --git a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java new file mode 100644 index 0000000000000..2ce82c1e79941 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.RemovePersistentTaskAction.Request; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; + +public class CancelPersistentTaskRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLength(10)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java new file mode 100644 index 0000000000000..1169ff91e1308 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -0,0 +1,453 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static java.util.Collections.emptyMap; +import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class PersistentTasksClusterServiceTests extends ESTestCase { + + public void testReassignmentRequired() { + int numberOfIterations = randomIntBetween(1, 30); + ClusterState clusterState = initialState(); + for (int i = 0; i < numberOfIterations; i++) { + boolean significant = randomBoolean(); + ClusterState previousState = clusterState; + logger.info("inter {} significant: {}", i, significant); + if (significant) { + clusterState = significantChange(clusterState); + } else { + clusterState = insignificantChange(clusterState); + } + ClusterChangedEvent event = new ClusterChangedEvent("test", clusterState, previousState); + assertThat(dumpEvent(event), PersistentTasksClusterService.reassignmentRequired(event, + new PersistentTasksClusterService.ExecutorNodeDecider() { + @Override + public Assignment getAssignment( + String action, ClusterState currentState, Params params) { + if ("never_assign".equals(((TestParams) params).getTestParam())) { + return NO_NODE_FOUND; + } + return randomNodeAssignment(currentState.nodes()); + } + }), equalTo(significant)); + } + } + + public void testReassignTasksWithNoTasks() { + ClusterState clusterState = initialState(); + assertThat(reassign(clusterState).metaData().custom(PersistentTasksCustomMetaData.TYPE), nullValue()); + } + + public void testReassignConsidersClusterStateUpdates() { + ClusterState clusterState = initialState(); + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder( + clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)); + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); + addTestNodes(nodes, randomIntBetween(1, 10)); + int numberOfTasks = randomIntBetween(2, 40); + for (int i = 0; i < numberOfTasks; i++) { + addTask(tasks, "assign_one", randomBoolean() ? null : "no_longer_exits"); + } + + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); + clusterState = builder.metaData(metaData).nodes(nodes).build(); + ClusterState newClusterState = reassign(clusterState); + + PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + assertThat(tasksInProgress, notNullValue()); + + } + + public void testReassignTasks() { + ClusterState clusterState = initialState(); + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder( + clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE)); + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()); + addTestNodes(nodes, randomIntBetween(1, 10)); + int numberOfTasks = randomIntBetween(0, 40); + for (int i = 0; i < numberOfTasks; i++) { + switch (randomInt(2)) { + case 0: + // add an unassigned task that should get assigned because it's assigned to a non-existing node or unassigned + addTask(tasks, "assign_me", randomBoolean() ? null : "no_longer_exits"); + break; + case 1: + // add a task assigned to non-existing node that should not get assigned + addTask(tasks, "dont_assign_me", randomBoolean() ? null : "no_longer_exits"); + break; + case 2: + addTask(tasks, "assign_one", randomBoolean() ? null : "no_longer_exits"); + break; + + } + } + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); + clusterState = builder.metaData(metaData).nodes(nodes).build(); + ClusterState newClusterState = reassign(clusterState); + + PersistentTasksCustomMetaData tasksInProgress = newClusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + assertThat(tasksInProgress, notNullValue()); + + assertThat("number of tasks shouldn't change as a result or reassignment", + numberOfTasks, equalTo(tasksInProgress.tasks().size())); + + int assignOneCount = 0; + + for (PersistentTask task : tasksInProgress.tasks()) { + // explanation should correspond to the action name + switch (((TestParams) task.getParams()).getTestParam()) { + case "assign_me": + assertThat(task.getExecutorNode(), notNullValue()); + assertThat(task.isAssigned(), equalTo(true)); + if (clusterState.nodes().nodeExists(task.getExecutorNode()) == false) { + logger.info(clusterState.metaData().custom(PersistentTasksCustomMetaData.TYPE).toString()); + } + assertThat("task should be assigned to a node that is in the cluster, was assigned to " + task.getExecutorNode(), + clusterState.nodes().nodeExists(task.getExecutorNode()), equalTo(true)); + assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + break; + case "dont_assign_me": + assertThat(task.getExecutorNode(), nullValue()); + assertThat(task.isAssigned(), equalTo(false)); + assertThat(task.getAssignment().getExplanation(), equalTo("no appropriate nodes found for the assignment")); + break; + case "assign_one": + if (task.isAssigned()) { + assignOneCount++; + assertThat("more than one assign_one tasks are assigned", assignOneCount, lessThanOrEqualTo(1)); + assertThat(task.getAssignment().getExplanation(), equalTo("test assignment")); + } else { + assertThat(task.getAssignment().getExplanation(), equalTo("only one task can be assigned at a time")); + } + break; + default: + fail("Unknown action " + task.getTaskName()); + } + } + } + + + private void addTestNodes(DiscoveryNodes.Builder nodes, int nonLocalNodesCount) { + for (int i = 0; i < nonLocalNodesCount; i++) { + nodes.add(new DiscoveryNode("other_node_" + i, buildNewFakeTransportAddress(), Version.CURRENT)); + } + } + + private ClusterState reassign(ClusterState clusterState) { + return PersistentTasksClusterService.reassignTasks(clusterState, logger, + new PersistentTasksClusterService.ExecutorNodeDecider() { + @Override + public Assignment getAssignment( + String action, ClusterState currentState, Params params) { + TestParams testParams = (TestParams) params; + switch (testParams.getTestParam()) { + case "assign_me": + return randomNodeAssignment(currentState.nodes()); + case "dont_assign_me": + return NO_NODE_FOUND; + case "fail_me_if_called": + fail("the decision decider shouldn't be called on this task"); + return null; + case "assign_one": + return assignOnlyOneTaskAtATime(currentState); + default: + fail("unknown param " + testParams.getTestParam()); + } + return NO_NODE_FOUND; + } + }); + + } + + private Assignment assignOnlyOneTaskAtATime(ClusterState clusterState) { + DiscoveryNodes nodes = clusterState.nodes(); + PersistentTasksCustomMetaData tasksInProgress = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + if (tasksInProgress.findTasks(TestPersistentTasksExecutor.NAME, task -> + "assign_one".equals(((TestParams) task.getParams()).getTestParam()) && + nodes.nodeExists(task.getExecutorNode())).isEmpty()) { + return randomNodeAssignment(clusterState.nodes()); + } else { + return new Assignment(null, "only one task can be assigned at a time"); + } + } + + private Assignment randomNodeAssignment(DiscoveryNodes nodes) { + if (nodes.getNodes().isEmpty()) { + return NO_NODE_FOUND; + } + List nodeList = new ArrayList<>(); + for (ObjectCursor node : nodes.getNodes().keys()) { + nodeList.add(node.value); + } + String node = randomFrom(nodeList); + if (node != null) { + return new Assignment(node, "test assignment"); + } else { + return NO_NODE_FOUND; + } + } + + private String dumpEvent(ClusterChangedEvent event) { + return "nodes_changed: " + event.nodesChanged() + + " nodes_removed:" + event.nodesRemoved() + + " routing_table_changed:" + event.routingTableChanged() + + " tasks: " + event.state().metaData().custom(PersistentTasksCustomMetaData.TYPE); + } + + private ClusterState significantChange(ClusterState clusterState) { + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + if (tasks != null) { + if (randomBoolean()) { + for (PersistentTask task : tasks.tasks()) { + if (task.isAssigned() && clusterState.nodes().nodeExists(task.getExecutorNode())) { + logger.info("removed node {}", task.getExecutorNode()); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(task.getExecutorNode())); + return builder.build(); + } + } + } + } + boolean tasksOrNodesChanged = false; + // add a new unassigned task + if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { + // we don't have any unassigned tasks - add some + if (randomBoolean()) { + logger.info("added random task"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), null); + tasksOrNodesChanged = true; + } else { + logger.info("added unassignable task with custom assignment message"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), PersistentTasksCustomMetaData.builder(tasks), + new Assignment(null, "change me"), "never_assign"); + tasksOrNodesChanged = true; + } + } + // add a node if there are unassigned tasks + if (clusterState.nodes().getNodes().isEmpty()) { + logger.info("added random node"); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAlphaOfLength(10)))); + tasksOrNodesChanged = true; + } + + if (tasksOrNodesChanged == false) { + // change routing table to simulate a change + logger.info("changed routing table"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); + RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); + changeRoutingTable(metaData, routingTable); + builder.metaData(metaData).routingTable(routingTable.build()); + } + return builder.build(); + } + + private PersistentTasksCustomMetaData removeTasksWithChangingAssignment(PersistentTasksCustomMetaData tasks) { + if (tasks != null) { + boolean changed = false; + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); + for (PersistentTask task : tasks.tasks()) { + // Remove all unassigned tasks that cause changing assignments they might trigger a significant change + if ("never_assign".equals(((TestParams) task.getParams()).getTestParam()) && + "change me".equals(task.getAssignment().getExplanation())) { + logger.info("removed task with changing assignment {}", task.getId()); + tasksBuilder.removeTask(task.getId()); + changed = true; + } + } + if (changed) { + return tasksBuilder.build(); + } + } + return tasks; + } + + private ClusterState insignificantChange(ClusterState clusterState) { + ClusterState.Builder builder = ClusterState.builder(clusterState); + PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + tasks = removeTasksWithChangingAssignment(tasks); + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); + + if (randomBoolean()) { + if (hasAssignableTasks(tasks, clusterState.nodes()) == false) { + // we don't have any unassigned tasks - adding a node or changing a routing table shouldn't affect anything + if (randomBoolean()) { + logger.info("added random node"); + builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode(randomAlphaOfLength(10)))); + } + if (randomBoolean()) { + logger.info("added random unassignable task"); + addRandomTask(builder, MetaData.builder(clusterState.metaData()), tasksBuilder, NO_NODE_FOUND, "never_assign"); + return builder.build(); + } + logger.info("changed routing table"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); + RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable()); + changeRoutingTable(metaData, routingTable); + builder.metaData(metaData).routingTable(routingTable.build()); + return builder.build(); + } + } + if (randomBoolean()) { + // remove a node that doesn't have any tasks assigned to it and it's not the master node + for (DiscoveryNode node : clusterState.nodes()) { + if (hasTasksAssignedTo(tasks, node.getId()) == false && "this_node".equals(node.getId()) == false) { + logger.info("removed unassigned node {}", node.getId()); + return builder.nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(node.getId())).build(); + } + } + } + + if (randomBoolean()) { + // clear the task + if (randomBoolean()) { + logger.info("removed all tasks"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData.builder().build()); + return builder.metaData(metaData).build(); + } else { + logger.info("set task custom to null"); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).removeCustom(PersistentTasksCustomMetaData.TYPE); + return builder.metaData(metaData).build(); + } + } + logger.info("removed all unassigned tasks and changed routing table"); + if (tasks != null) { + for (PersistentTask task : tasks.tasks()) { + if (task.getExecutorNode() == null || "never_assign".equals(((TestParams) task.getParams()).getTestParam())) { + tasksBuilder.removeTask(task.getId()); + } + } + } + // Just add a random index - that shouldn't change anything + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + MetaData.Builder metaData = MetaData.builder(clusterState.metaData()).put(indexMetaData, false) + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()); + return builder.metaData(metaData).build(); + } + + private boolean hasAssignableTasks(PersistentTasksCustomMetaData tasks, DiscoveryNodes discoveryNodes) { + if (tasks == null || tasks.tasks().isEmpty()) { + return false; + } + return tasks.tasks().stream().anyMatch(task -> { + if (task.getExecutorNode() == null || discoveryNodes.nodeExists(task.getExecutorNode())) { + return "never_assign".equals(((TestParams) task.getParams()).getTestParam()) == false; + } + return false; + }); + } + + private boolean hasTasksAssignedTo(PersistentTasksCustomMetaData tasks, String nodeId) { + return tasks != null && tasks.tasks().stream().anyMatch( + task -> nodeId.equals(task.getExecutorNode())) == false; + } + + private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, + MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, + String node) { + return addRandomTask(clusterStateBuilder, metaData, tasks, new Assignment(node, randomAlphaOfLength(10)), + randomAlphaOfLength(10)); + } + + private ClusterState.Builder addRandomTask(ClusterState.Builder clusterStateBuilder, + MetaData.Builder metaData, PersistentTasksCustomMetaData.Builder tasks, + Assignment assignment, String param) { + return clusterStateBuilder.metaData(metaData.putCustom(PersistentTasksCustomMetaData.TYPE, + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams(param), assignment).build())); + } + + private void addTask(PersistentTasksCustomMetaData.Builder tasks, String param, String node) { + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams(param), + new Assignment(node, "explanation: " + param)); + } + + private DiscoveryNode newNode(String nodeId) { + return new DiscoveryNode(nodeId, buildNewFakeTransportAddress(), emptyMap(), + Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), + Version.CURRENT); + } + + + private ClusterState initialState() { + MetaData.Builder metaData = MetaData.builder(); + RoutingTable.Builder routingTable = RoutingTable.builder(); + int randomIndices = randomIntBetween(0, 5); + for (int i = 0; i < randomIndices; i++) { + changeRoutingTable(metaData, routingTable); + } + + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); + nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), "this_node")); + nodes.localNodeId("this_node"); + nodes.masterNodeId("this_node"); + + return ClusterState.builder(ClusterName.DEFAULT) + .metaData(metaData) + .routingTable(routingTable.build()) + .build(); + } + + private void changeRoutingTable(MetaData.Builder metaData, RoutingTable.Builder routingTable) { + IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)) + .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random()))) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + metaData.put(indexMetaData, false); + routingTable.addAsNew(indexMetaData); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java new file mode 100644 index 0000000000000..7e731884dda41 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java @@ -0,0 +1,262 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaData.Custom; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.AbstractDiffableSerializationTestCase; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Builder; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; + +import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY; +import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT; +import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND; + +public class PersistentTasksCustomMetaDataTests extends AbstractDiffableSerializationTestCase { + + @Override + protected PersistentTasksCustomMetaData createTestInstance() { + int numberOfTasks = randomInt(10); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); + for (int i = 0; i < numberOfTasks; i++) { + String taskId = UUIDs.base64UUID(); + tasks.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams(randomAlphaOfLength(10)), + randomAssignment()); + if (randomBoolean()) { + // From time to time update status + tasks.updateTaskStatus(taskId, new Status(randomAlphaOfLength(10))); + } + } + return tasks.build(); + } + + @Override + protected Writeable.Reader instanceReader() { + return PersistentTasksCustomMetaData::new; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Arrays.asList( + new Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::new), + new Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, PersistentTasksCustomMetaData::readDiffFrom), + new Entry(PersistentTaskParams.class, TestPersistentTasksExecutor.NAME, TestParams::new), + new Entry(Task.Status.class, TestPersistentTasksExecutor.NAME, Status::new) + )); + } + + @Override + protected Custom makeTestChanges(Custom testInstance) { + Builder builder = PersistentTasksCustomMetaData.builder((PersistentTasksCustomMetaData) testInstance); + switch (randomInt(3)) { + case 0: + addRandomTask(builder); + break; + case 1: + if (builder.getCurrentTaskIds().isEmpty()) { + addRandomTask(builder); + } else { + builder.reassignTask(pickRandomTask(builder), randomAssignment()); + } + break; + case 2: + if (builder.getCurrentTaskIds().isEmpty()) { + addRandomTask(builder); + } else { + builder.updateTaskStatus(pickRandomTask(builder), randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); + } + break; + case 3: + if (builder.getCurrentTaskIds().isEmpty()) { + addRandomTask(builder); + } else { + builder.removeTask(pickRandomTask(builder)); + } + break; + } + return builder.build(); + } + + @Override + protected Writeable.Reader> diffReader() { + return PersistentTasksCustomMetaData::readDiffFrom; + } + + @Override + protected PersistentTasksCustomMetaData doParseInstance(XContentParser parser) throws IOException { + return PersistentTasksCustomMetaData.fromXContent(parser); + } + +/* + @Override + protected XContentBuilder toXContent(Custom instance, XContentType contentType) throws IOException { + return toXContent(instance, contentType, new ToXContent.MapParams( + Collections.singletonMap(MetaData.CONTEXT_MODE_PARAM, MetaData.XContentContext.API.toString()))); + } +*/ + + private String addRandomTask(Builder builder) { + String taskId = UUIDs.base64UUID(); + builder.addTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams(randomAlphaOfLength(10)), randomAssignment()); + return taskId; + } + + private String pickRandomTask(PersistentTasksCustomMetaData.Builder testInstance) { + return randomFrom(new ArrayList<>(testInstance.getCurrentTaskIds())); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return new NamedXContentRegistry(Arrays.asList( + new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(TestPersistentTasksExecutor.NAME), + TestParams::fromXContent), + new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(TestPersistentTasksExecutor.NAME), Status::fromXContent) + )); + } + + @SuppressWarnings("unchecked") + public void testSerializationContext() throws Exception { + PersistentTasksCustomMetaData testInstance = createTestInstance(); + for (int i = 0; i < randomInt(10); i++) { + testInstance = (PersistentTasksCustomMetaData) makeTestChanges(testInstance); + } + + ToXContent.MapParams params = new ToXContent.MapParams( + Collections.singletonMap(MetaData.CONTEXT_MODE_PARAM, randomFrom(CONTEXT_MODE_SNAPSHOT, CONTEXT_MODE_GATEWAY))); + + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference shuffled = toShuffledXContent(testInstance, xContentType, params, false); + + XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled); + PersistentTasksCustomMetaData newInstance = doParseInstance(parser); + assertNotSame(newInstance, testInstance); + + assertEquals(testInstance.tasks().size(), newInstance.tasks().size()); + for (PersistentTask testTask : testInstance.tasks()) { + PersistentTask newTask = (PersistentTask) newInstance.getTask(testTask.getId()); + assertNotNull(newTask); + + // Things that should be serialized + assertEquals(testTask.getTaskName(), newTask.getTaskName()); + assertEquals(testTask.getId(), newTask.getId()); + assertEquals(testTask.getStatus(), newTask.getStatus()); + assertEquals(testTask.getParams(), newTask.getParams()); + + // Things that shouldn't be serialized + assertEquals(0, newTask.getAllocationId()); + assertNull(newTask.getExecutorNode()); + } + } + + public void testBuilder() { + PersistentTasksCustomMetaData persistentTasks = null; + String lastKnownTask = ""; + for (int i = 0; i < randomIntBetween(10, 100); i++) { + final Builder builder; + if (randomBoolean()) { + builder = PersistentTasksCustomMetaData.builder(); + } else { + builder = PersistentTasksCustomMetaData.builder(persistentTasks); + } + boolean changed = false; + for (int j = 0; j < randomIntBetween(1, 10); j++) { + switch (randomInt(4)) { + case 0: + lastKnownTask = addRandomTask(builder); + changed = true; + break; + case 1: + if (builder.hasTask(lastKnownTask)) { + changed = true; + builder.reassignTask(lastKnownTask, randomAssignment()); + } else { + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.reassignTask(fLastKnownTask, randomAssignment())); + } + break; + case 2: + if (builder.hasTask(lastKnownTask)) { + changed = true; + builder.updateTaskStatus(lastKnownTask, randomBoolean() ? new Status(randomAlphaOfLength(10)) : null); + } else { + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.updateTaskStatus(fLastKnownTask, null)); + } + break; + case 3: + if (builder.hasTask(lastKnownTask)) { + changed = true; + builder.removeTask(lastKnownTask); + } else { + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.removeTask(fLastKnownTask)); + } + break; + case 4: + if (builder.hasTask(lastKnownTask)) { + changed = true; + builder.finishTask(lastKnownTask); + } else { + String fLastKnownTask = lastKnownTask; + expectThrows(ResourceNotFoundException.class, () -> builder.finishTask(fLastKnownTask)); + } + break; + } + } + assertEquals(changed, builder.isChanged()); + persistentTasks = builder.build(); + } + + } + + private Assignment randomAssignment() { + if (randomBoolean()) { + if (randomBoolean()) { + return NO_NODE_FOUND; + } else { + return new Assignment(null, randomAlphaOfLength(10)); + } + } + return new Assignment(randomAlphaOfLength(10), randomAlphaOfLength(10)); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java new file mode 100644 index 0000000000000..0dddaaa783906 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1) +public class PersistentTasksExecutorFullRestartIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(TestPersistentTasksPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + protected boolean ignoreExternalCluster() { + return true; + } + + @TestLogging("org.elasticsearch.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG") + public void testFullClusterRestart() throws Exception { + PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class); + int numberOfTasks = randomIntBetween(1, 10); + String[] taskIds = new String[numberOfTasks]; + List>> futures = new ArrayList<>(numberOfTasks); + + for (int i = 0; i < numberOfTasks; i++) { + PlainActionFuture> future = new PlainActionFuture<>(); + futures.add(future); + taskIds[i] = UUIDs.base64UUID(); + service.startPersistentTask(taskIds[i], TestPersistentTasksExecutor.NAME, randomBoolean() ? null : new TestParams("Blah"), + future); + } + + for (int i = 0; i < numberOfTasks; i++) { + assertThat(futures.get(i).get().getId(), equalTo(taskIds[i])); + } + + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); + assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); + + // Make sure that at least one of the tasks is running + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), greaterThan(0)); + }); + + // Restart cluster + internalCluster().fullRestart(); + ensureYellow(); + + tasksInProgress = internalCluster().clusterService().state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + assertThat(tasksInProgress.tasks().size(), equalTo(numberOfTasks)); + // Check that cluster state is correct + for (int i = 0; i < numberOfTasks; i++) { + PersistentTask task = tasksInProgress.getTask(taskIds[i]); + assertNotNull(task); + } + + logger.info("Waiting for {} tasks to start", numberOfTasks); + assertBusy(() -> { + // Wait for all tasks to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(numberOfTasks)); + }); + + logger.info("Complete all tasks"); + // Complete the running task and make sure it finishes properly + assertThat(new TestPersistentTasksPlugin.TestTasksRequestBuilder(client()).setOperation("finish").get().getTasks().size(), + equalTo(numberOfTasks)); + + assertBusy(() -> { + // Make sure the task is removed from the cluster state + assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty()); + }); + + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java new file mode 100644 index 0000000000000..91ccd8a37f06b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorIT.java @@ -0,0 +1,298 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.PersistentTasksService.WaitForPersistentTaskStatusListener; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestTasksRequestBuilder; +import org.junit.After; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, minNumDataNodes = 2) +public class PersistentTasksExecutorIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(TestPersistentTasksPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + protected boolean ignoreExternalCluster() { + return true; + } + + @After + public void cleanup() throws Exception { + assertNoRunningTasks(); + } + + public static class WaitForPersistentTaskStatusFuture + extends PlainActionFuture> + implements WaitForPersistentTaskStatusListener { + } + + public void testPersistentActionFailure() throws Exception { + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PlainActionFuture> future = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); + long allocationId = future.get().getAllocationId(); + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); + }); + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .get().getTasks().get(0); + logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); + // Verifying parent + assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(allocationId)); + assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); + + logger.info("Failing the running task"); + // Fail the running task and make sure it restarts properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("fail").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); + assertBusy(() -> { + // Wait for the task to disappear completely + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), + empty()); + }); + } + + public void testPersistentActionCompletion() throws Exception { + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PlainActionFuture> future = new PlainActionFuture<>(); + String taskId = UUIDs.base64UUID(); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); + long allocationId = future.get().getAllocationId(); + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); + }); + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .setDetailed(true).get().getTasks().get(0); + logger.info("Found running task with id {} and parent {}", firstRunningTask.getId(), firstRunningTask.getParentTaskId()); + // Verifying parent and description + assertThat(firstRunningTask.getParentTaskId().getId(), equalTo(allocationId)); + assertThat(firstRunningTask.getParentTaskId().getNodeId(), equalTo("cluster")); + assertThat(firstRunningTask.getDescription(), equalTo("id=" + taskId)); + + if (randomBoolean()) { + logger.info("Simulating errant completion notification"); + //try sending completion request with incorrect allocation id + PlainActionFuture> failedCompletionNotificationFuture = new PlainActionFuture<>(); + persistentTasksService.sendCompletionNotification(taskId, Long.MAX_VALUE, null, failedCompletionNotificationFuture); + assertThrows(failedCompletionNotificationFuture, ResourceNotFoundException.class); + // Make sure that the task is still running + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .setDetailed(true).get().getTasks().size(), equalTo(1)); + } + + stopOrCancelTask(firstRunningTask.getTaskId()); + } + + public void testPersistentActionWithNoAvailableNode() throws Exception { + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PlainActionFuture> future = new PlainActionFuture<>(); + TestParams testParams = new TestParams("Blah"); + testParams.setExecutorNodeAttr("test"); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, testParams, future); + String taskId = future.get().getId(); + + Settings nodeSettings = Settings.builder().put(nodeSettings(0)).put("node.attr.test_attr", "test").build(); + String newNode = internalCluster().startNode(nodeSettings); + String newNodeId = internalCluster().clusterService(newNode).localNode().getId(); + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() + .size(), equalTo(1)); + }); + TaskInfo taskInfo = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .get().getTasks().get(0); + + // Verifying the the task runs on the new node + assertThat(taskInfo.getTaskId().getNodeId(), equalTo(newNodeId)); + + internalCluster().stopRandomNode(settings -> "test".equals(settings.get("node.attr.test_attr"))); + + assertBusy(() -> { + // Wait for the task to disappear completely + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), + empty()); + }); + + // Remove the persistent task + PlainActionFuture> removeFuture = new PlainActionFuture<>(); + persistentTasksService.cancelPersistentTask(taskId, removeFuture); + assertEquals(removeFuture.get().getId(), taskId); + } + + public void testPersistentActionStatusUpdate() throws Exception { + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PlainActionFuture> future = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); + String taskId = future.get().getId(); + + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks() + .size(), equalTo(1)); + }); + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .get().getTasks().get(0); + + PersistentTasksCustomMetaData tasksInProgress = internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE); + assertThat(tasksInProgress.tasks().size(), equalTo(1)); + assertThat(tasksInProgress.tasks().iterator().next().getStatus(), nullValue()); + + int numberOfUpdates = randomIntBetween(1, 10); + for (int i = 0; i < numberOfUpdates; i++) { + logger.info("Updating the task status"); + // Complete the running task and make sure it finishes properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("update_status").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + int finalI = i; + WaitForPersistentTaskStatusFuture future1 = new WaitForPersistentTaskStatusFuture<>(); + persistentTasksService.waitForPersistentTaskStatus(taskId, + task -> task != null && task.getStatus() != null && task.getStatus().toString() != null && + task.getStatus().toString().equals("{\"phase\":\"phase " + (finalI + 1) + "\"}"), + TimeValue.timeValueSeconds(10), future1); + assertThat(future1.get().getId(), equalTo(taskId)); + } + + WaitForPersistentTaskStatusFuture future1 = new WaitForPersistentTaskStatusFuture<>(); + persistentTasksService.waitForPersistentTaskStatus(taskId, + task -> false, TimeValue.timeValueMillis(10), future1); + + assertThrows(future1, IllegalStateException.class, "timed out after 10ms"); + + PlainActionFuture> failedUpdateFuture = new PlainActionFuture<>(); + persistentTasksService.updateStatus(taskId, -2, new Status("should fail"), failedUpdateFuture); + assertThrows(failedUpdateFuture, ResourceNotFoundException.class, "the task with id " + taskId + + " and allocation id -2 doesn't exist"); + + // Wait for the task to disappear + WaitForPersistentTaskStatusFuture future2 = new WaitForPersistentTaskStatusFuture<>(); + persistentTasksService.waitForPersistentTaskStatus(taskId, Objects::isNull, TimeValue.timeValueSeconds(10), future2); + + logger.info("Completing the running task"); + // Complete the running task and make sure it finishes properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + assertThat(future2.get(), nullValue()); + } + + public void testCreatePersistentTaskWithDuplicateId() throws Exception { + PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class); + PlainActionFuture> future = new PlainActionFuture<>(); + String taskId = UUIDs.base64UUID(); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future); + future.get(); + + PlainActionFuture> future2 = new PlainActionFuture<>(); + persistentTasksService.startPersistentTask(taskId, TestPersistentTasksExecutor.NAME, new TestParams("Blah"), future2); + assertThrows(future2, ResourceAlreadyExistsException.class); + + assertBusy(() -> { + // Wait for the task to start + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks().size(), equalTo(1)); + }); + + TaskInfo firstRunningTask = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]") + .get().getTasks().get(0); + + logger.info("Completing the running task"); + // Fail the running task and make sure it restarts properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(firstRunningTask.getTaskId()) + .get().getTasks().size(), equalTo(1)); + + logger.info("Waiting for persistent task with id {} to disappear", firstRunningTask.getId()); + assertBusy(() -> { + // Wait for the task to disappear completely + assertThat(client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get().getTasks(), + empty()); + }); + } + + private void stopOrCancelTask(TaskId taskId) { + if (randomBoolean()) { + logger.info("Completing the running task"); + // Complete the running task and make sure it finishes properly + assertThat(new TestTasksRequestBuilder(client()).setOperation("finish").setTaskId(taskId) + .get().getTasks().size(), equalTo(1)); + + } else { + logger.info("Cancelling the running task"); + // Cancel the running task and make sure it finishes properly + assertThat(client().admin().cluster().prepareCancelTasks().setTaskId(taskId) + .get().getTasks().size(), equalTo(1)); + } + + + } + + private void assertNoRunningTasks() throws Exception { + assertBusy(() -> { + // Wait for the task to finish + List tasks = client().admin().cluster().prepareListTasks().setActions(TestPersistentTasksExecutor.NAME + "[c]").get() + .getTasks(); + logger.info("Found {} tasks", tasks.size()); + assertThat(tasks.size(), equalTo(0)); + + // Make sure the task is removed from the cluster state + assertThat(((PersistentTasksCustomMetaData) internalCluster().clusterService().state().getMetaData() + .custom(PersistentTasksCustomMetaData.TYPE)).tasks(), empty()); + }); + } + +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java new file mode 100644 index 0000000000000..ada0e24baa7bd --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; + +import java.util.Collections; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength; + +public class PersistentTasksExecutorResponseTests extends AbstractStreamableTestCase { + + @Override + protected PersistentTaskResponse createTestInstance() { + if (randomBoolean()) { + return new PersistentTaskResponse( + new PersistentTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, + new TestPersistentTasksPlugin.TestParams("test"), + randomLong(), PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT)); + } else { + return new PersistentTaskResponse(null); + } + } + + @Override + protected PersistentTaskResponse createBlankInstance() { + return new PersistentTaskResponse(); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.singletonList( + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, + TestPersistentTasksExecutor.NAME, TestPersistentTasksPlugin.TestParams::new) + )); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java new file mode 100644 index 0000000000000..37925b7144fa5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.persistent.PersistentTasksNodeService.Status; + +import static org.hamcrest.Matchers.containsString; + +public class PersistentTasksNodeServiceStatusTests extends AbstractWireSerializingTestCase { + + @Override + protected Status createTestInstance() { + return new Status(randomFrom(AllocatedPersistentTask.State.values())); + } + + @Override + protected Writeable.Reader instanceReader() { + return Status::new; + } + + public void testToString() { + assertThat(createTestInstance().toString(), containsString("state")); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java new file mode 100644 index 0000000000000..f9a70637e502f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -0,0 +1,372 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.sameInstance; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class PersistentTasksNodeServiceTests extends ESTestCase { + + private ThreadPool threadPool; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + threadPool = new TestThreadPool(getClass().getName()); + } + + + @Override + @After + public void tearDown() throws Exception { + terminate(threadPool); + super.tearDown(); + } + + private ClusterState createInitialClusterState(int nonLocalNodesCount, Settings settings) { + ClusterState.Builder state = ClusterState.builder(new ClusterName("PersistentActionExecutorTests")); + state.metaData(MetaData.builder().generateClusterUuidIfNeeded()); + state.routingTable(RoutingTable.builder().build()); + DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); + nodes.add(DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), "this_node")); + for (int i = 0; i < nonLocalNodesCount; i++) { + nodes.add(new DiscoveryNode("other_node_" + i, buildNewFakeTransportAddress(), Version.CURRENT)); + } + nodes.localNodeId("this_node"); + state.nodes(nodes); + return state.build(); + } + + public void testStartTask() throws Exception { + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); + int nonLocalNodesCount = randomInt(10); + // need to account for 5 original tasks on each node and their relocations + for (int i = 0; i < (nonLocalNodesCount + 1) * 10; i++) { + TaskId parentId = new TaskId("cluster", i); + when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any(), any())).thenReturn( + new TestPersistentTasksPlugin.TestTask(i, "persistent", "test", "", parentId, Collections.emptyMap())); + } + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); + + MockExecutor executor = new MockExecutor(); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), executor); + + ClusterState state = createInitialClusterState(nonLocalNodesCount, Settings.EMPTY); + + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); + boolean added = false; + if (nonLocalNodesCount > 0) { + for (int i = 0; i < randomInt(5); i++) { + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("other_" + i), + new Assignment("other_node_" + randomInt(nonLocalNodesCount), "test assignment on other node")); + if (added == false && randomBoolean()) { + added = true; + tasks.addTask(UUIDs.base64UUID(), TestPersistentTasksExecutor.NAME, new TestParams("this_param"), + new Assignment("this_node", "test assignment on this node")); + } + } + } + + if (added == false) { + logger.info("No local node action was added"); + + } + MetaData.Builder metaData = MetaData.builder(state.metaData()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); + ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); + + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + if (added) { + // Action for this node was added, let's make sure it was invoked + assertThat(executor.executions.size(), equalTo(1)); + + // Add task on some other node + state = newClusterState; + newClusterState = addTask(state, TestPersistentTasksExecutor.NAME, null, "some_other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action wasn't called again + assertThat(executor.executions.size(), equalTo(1)); + + // Start another task on this node + state = newClusterState; + newClusterState = addTask(state, TestPersistentTasksExecutor.NAME, new TestParams("this_param"), "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action was called this time + assertThat(executor.size(), equalTo(2)); + + // Finish both tasks + executor.get(0).task.markAsFailed(new RuntimeException()); + executor.get(1).task.markAsCompleted(); + String failedTaskId = executor.get(0).task.getPersistentTaskId(); + String finishedTaskId = executor.get(1).task.getPersistentTaskId(); + executor.clear(); + + // Add task on some other node + state = newClusterState; + newClusterState = addTask(state, TestPersistentTasksExecutor.NAME, null, "some_other_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action wasn't called again + assertThat(executor.size(), equalTo(0)); + + // Simulate reallocation of the failed task on the same node + state = newClusterState; + newClusterState = reallocateTask(state, failedTaskId, "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Simulate removal of the finished task + state = newClusterState; + newClusterState = removeTask(state, finishedTaskId); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure action was only allocated on this node once + assertThat(executor.size(), equalTo(1)); + } + + } + + public void testParamsStatusAndNodeTaskAreDelegated() throws Exception { + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getTaskName()).thenReturn(TestPersistentTasksExecutor.NAME); + TaskId parentId = new TaskId("cluster", 1); + AllocatedPersistentTask nodeTask = + new TestPersistentTasksPlugin.TestTask(0, "persistent", "test", "", parentId, Collections.emptyMap()); + when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any(), any())).thenReturn(nodeTask); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); + + MockExecutor executor = new MockExecutor(); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), executor); + + ClusterState state = createInitialClusterState(1, Settings.EMPTY); + + Task.Status status = new TestPersistentTasksPlugin.Status("_test_phase"); + PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder(); + String taskId = UUIDs.base64UUID(); + TestParams taskParams = new TestParams("other_0"); + tasks.addTask(taskId, TestPersistentTasksExecutor.NAME, taskParams, + new Assignment("this_node", "test assignment on other node")); + tasks.updateTaskStatus(taskId, status); + MetaData.Builder metaData = MetaData.builder(state.metaData()); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks.build()); + ClusterState newClusterState = ClusterState.builder(state).metaData(metaData).build(); + + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + assertThat(executor.size(), equalTo(1)); + assertThat(executor.get(0).params, sameInstance(taskParams)); + assertThat(executor.get(0).status, sameInstance(status)); + assertThat(executor.get(0).task, sameInstance(nodeTask)); + } + + public void testTaskCancellation() { + AtomicLong capturedTaskId = new AtomicLong(); + AtomicReference> capturedListener = new AtomicReference<>(); + PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null, null) { + @Override + public void sendTaskManagerCancellation(long taskId, ActionListener listener) { + capturedTaskId.set(taskId); + capturedListener.set(listener); + } + + @Override + public void sendCompletionNotification(String taskId, long allocationId, Exception failure, + ActionListener> listener) { + fail("Shouldn't be called during Cluster State cancellation"); + } + }; + @SuppressWarnings("unchecked") PersistentTasksExecutor action = mock(PersistentTasksExecutor.class); + when(action.getExecutor()).thenReturn(ThreadPool.Names.SAME); + when(action.getTaskName()).thenReturn("test"); + when(action.createTask(anyLong(), anyString(), anyString(), any(), any(), any())) + .thenReturn(new TestPersistentTasksPlugin.TestTask(1, "persistent", "test", "", new TaskId("cluster", 1), + Collections.emptyMap())); + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action)); + + int nonLocalNodesCount = randomInt(10); + MockExecutor executor = new MockExecutor(); + TaskManager taskManager = new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()); + PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService, + registry, taskManager, executor); + + ClusterState state = createInitialClusterState(nonLocalNodesCount, Settings.EMPTY); + + ClusterState newClusterState = state; + // Allocate first task + state = newClusterState; + newClusterState = addTask(state, "test", null, "this_node"); + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Check the the task is know to the task manager + assertThat(taskManager.getTasks().size(), equalTo(1)); + AllocatedPersistentTask runningTask = (AllocatedPersistentTask)taskManager.getTasks().values().iterator().next(); + String persistentId = runningTask.getPersistentTaskId(); + long localId = runningTask.getId(); + // Make sure it returns correct status + Task.Status status = runningTask.getStatus(); + assertThat(status.toString(), equalTo("{\"state\":\"STARTED\"}")); + + state = newClusterState; + // Relocate the task to some other node or remove it completely + if (randomBoolean()) { + newClusterState = reallocateTask(state, persistentId, "some_other_node"); + } else { + newClusterState = removeTask(state, persistentId); + } + coordinator.clusterChanged(new ClusterChangedEvent("test", newClusterState, state)); + + // Make sure it returns correct status + assertThat(taskManager.getTasks().size(), equalTo(1)); + assertThat(taskManager.getTasks().values().iterator().next().getStatus().toString(), equalTo("{\"state\":\"PENDING_CANCEL\"}")); + + + // That should trigger cancellation request + assertThat(capturedTaskId.get(), equalTo(localId)); + // Notify successful cancellation + capturedListener.get().onResponse(new CancelTasksResponse()); + + // finish or fail task + if (randomBoolean()) { + executor.get(0).task.markAsCompleted(); + } else { + executor.get(0).task.markAsFailed(new IOException("test")); + } + + // Check the the task is now removed from task manager + assertThat(taskManager.getTasks().values(), empty()); + + } + + private ClusterState addTask(ClusterState state, String action, Params params, + String node) { + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, + builder.addTask(UUIDs.base64UUID(), action, params, new Assignment(node, "test assignment")).build())).build(); + } + + private ClusterState reallocateTask(ClusterState state, String taskId, String node) { + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); + assertTrue(builder.hasTask(taskId)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, + builder.reassignTask(taskId, new Assignment(node, "test assignment")).build())).build(); + } + + private ClusterState removeTask(ClusterState state, String taskId) { + PersistentTasksCustomMetaData.Builder builder = + PersistentTasksCustomMetaData.builder(state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE)); + assertTrue(builder.hasTask(taskId)); + return ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, + builder.removeTask(taskId).build())).build(); + } + + private class Execution { + private final PersistentTaskParams params; + private final AllocatedPersistentTask task; + private final Task.Status status; + private final PersistentTasksExecutor holder; + + Execution(PersistentTaskParams params, AllocatedPersistentTask task, Task.Status status, PersistentTasksExecutor holder) { + this.params = params; + this.task = task; + this.status = status; + this.holder = holder; + } + } + + private class MockExecutor extends NodePersistentTasksExecutor { + private List executions = new ArrayList<>(); + + MockExecutor() { + super(null); + } + + @Override + public void executeTask(Params params, + Task.Status status, + AllocatedPersistentTask task, + PersistentTasksExecutor executor) { + executions.add(new Execution(params, task, status, executor)); + } + + public Execution get(int i) { + return executions.get(i); + } + + public int size() { + return executions.size(); + } + + public void clear() { + executions.clear(); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java new file mode 100644 index 0000000000000..3ce29d543d4b9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/RestartPersistentTaskRequestTests.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.CompletionPersistentTaskAction.Request; + +public class RestartPersistentTaskRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAlphaOfLength(10), randomLong(), null); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java new file mode 100644 index 0000000000000..3b0fc2a3d0495 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.persistent.StartPersistentTaskAction.Request; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.util.Collections; + +public class StartPersistentActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + TestParams testParams; + if (randomBoolean()) { + testParams = new TestParams(); + if (randomBoolean()) { + testParams.setTestParam(randomAlphaOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + testParams.setExecutorNodeAttr(randomAlphaOfLengthBetween(1, 20)); + } + } else { + testParams = null; + } + return new Request(UUIDs.base64UUID(), randomAlphaOfLengthBetween(1, 20), testParams); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.singletonList( + new Entry(PersistentTaskParams.class, TestPersistentTasksExecutor.NAME, TestParams::new) + )); + } +} diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java new file mode 100644 index 0000000000000..ca3e840028cb6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -0,0 +1,538 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.TaskOperationFailure; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksRequest; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.action.support.tasks.TasksRequestBuilder; +import org.elasticsearch.action.support.tasks.TransportTasksAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.PersistentTaskPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static java.util.Objects.requireNonNull; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.test.ESTestCase.awaitBusy; +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * A plugin that adds a test persistent task. + */ +public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin, PersistentTaskPlugin { + + @Override + public List> getActions() { + return Collections.singletonList(new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class)); + } + + @Override + public List> getPersistentTasksExecutor(ClusterService clusterService) { + return Collections.singletonList(new TestPersistentTasksExecutor(Settings.EMPTY, clusterService)); + } + + @Override + public List getNamedWriteables() { + return Arrays.asList( + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TestPersistentTasksExecutor.NAME, TestParams::new), + new NamedWriteableRegistry.Entry(Task.Status.class, + PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new), + new NamedWriteableRegistry.Entry(MetaData.Custom.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksCustomMetaData.TYPE, + PersistentTasksCustomMetaData::readDiffFrom), + new NamedWriteableRegistry.Entry(Task.Status.class, TestPersistentTasksExecutor.NAME, Status::new) + ); + } + + @Override + public List getNamedXContent() { + return Arrays.asList( + new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(PersistentTasksCustomMetaData.TYPE), + PersistentTasksCustomMetaData::fromXContent), + new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(TestPersistentTasksExecutor.NAME), + TestParams::fromXContent), + new NamedXContentRegistry.Entry(Task.Status.class, new ParseField(TestPersistentTasksExecutor.NAME), Status::fromXContent) + ); + } + + public static class TestParams implements PersistentTaskParams { + + public static final ConstructingObjectParser REQUEST_PARSER = + new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new TestParams((String) args[0])); + + static { + REQUEST_PARSER.declareString(constructorArg(), new ParseField("param")); + } + + private String executorNodeAttr = null; + + private String responseNode = null; + + private String testParam = null; + + public TestParams() { + + } + + public TestParams(String testParam) { + this.testParam = testParam; + } + + public TestParams(StreamInput in) throws IOException { + executorNodeAttr = in.readOptionalString(); + responseNode = in.readOptionalString(); + testParam = in.readOptionalString(); + } + + @Override + public String getWriteableName() { + return TestPersistentTasksExecutor.NAME; + } + + public void setExecutorNodeAttr(String executorNodeAttr) { + this.executorNodeAttr = executorNodeAttr; + } + + public void setTestParam(String testParam) { + this.testParam = testParam; + } + + public String getExecutorNodeAttr() { + return executorNodeAttr; + } + + public String getTestParam() { + return testParam; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(executorNodeAttr); + out.writeOptionalString(responseNode); + out.writeOptionalString(testParam); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("param", testParam); + builder.endObject(); + return builder; + } + + public static TestParams fromXContent(XContentParser parser) throws IOException { + return REQUEST_PARSER.parse(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestParams that = (TestParams) o; + return Objects.equals(executorNodeAttr, that.executorNodeAttr) && + Objects.equals(responseNode, that.responseNode) && + Objects.equals(testParam, that.testParam); + } + + @Override + public int hashCode() { + return Objects.hash(executorNodeAttr, responseNode, testParam); + } + } + + public static class Status implements Task.Status { + + private final String phase; + + public static final ConstructingObjectParser STATUS_PARSER = + new ConstructingObjectParser<>(TestPersistentTasksExecutor.NAME, args -> new Status((String) args[0])); + + static { + STATUS_PARSER.declareString(constructorArg(), new ParseField("phase")); + } + + public Status(String phase) { + this.phase = requireNonNull(phase, "Phase cannot be null"); + } + + public Status(StreamInput in) throws IOException { + phase = in.readString(); + } + + @Override + public String getWriteableName() { + return TestPersistentTasksExecutor.NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("phase", phase); + builder.endObject(); + return builder; + } + + public static Task.Status fromXContent(XContentParser parser) throws IOException { + return STATUS_PARSER.parse(parser, null); + } + + + @Override + public boolean isFragment() { + return false; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(phase); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + // Implements equals and hashcode for testing + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != Status.class) { + return false; + } + Status other = (Status) obj; + return phase.equals(other.phase); + } + + @Override + public int hashCode() { + return phase.hashCode(); + } + } + + + public static class TestPersistentTasksExecutor extends PersistentTasksExecutor { + + public static final String NAME = "cluster:admin/persistent/test"; + private final ClusterService clusterService; + + public TestPersistentTasksExecutor(Settings settings, ClusterService clusterService) { + super(settings, NAME, ThreadPool.Names.GENERIC); + this.clusterService = clusterService; + } + + @Override + public Assignment getAssignment(TestParams params, ClusterState clusterState) { + if (params == null || params.getExecutorNodeAttr() == null) { + return super.getAssignment(params, clusterState); + } else { + DiscoveryNode executorNode = selectLeastLoadedNode(clusterState, + discoveryNode -> params.getExecutorNodeAttr().equals(discoveryNode.getAttributes().get("test_attr"))); + if (executorNode != null) { + return new Assignment(executorNode.getId(), "test assignment"); + } else { + return NO_NODE_FOUND; + } + + } + } + + @Override + protected void nodeOperation(AllocatedPersistentTask task, TestParams params, Task.Status status) { + logger.info("started node operation for the task {}", task); + try { + TestTask testTask = (TestTask) task; + AtomicInteger phase = new AtomicInteger(); + while (true) { + // wait for something to happen + assertTrue(awaitBusy(() -> testTask.isCancelled() || + testTask.getOperation() != null || + clusterService.lifecycleState() != Lifecycle.State.STARTED, // speedup finishing on closed nodes + 30, TimeUnit.SECONDS)); // This can take a while during large cluster restart + if (clusterService.lifecycleState() != Lifecycle.State.STARTED) { + return; + } + if ("finish".equals(testTask.getOperation())) { + task.markAsCompleted(); + return; + } else if ("fail".equals(testTask.getOperation())) { + task.markAsFailed(new RuntimeException("Simulating failure")); + return; + } else if ("update_status".equals(testTask.getOperation())) { + testTask.setOperation(null); + CountDownLatch latch = new CountDownLatch(1); + Status newStatus = new Status("phase " + phase.incrementAndGet()); + logger.info("updating the task status to {}", newStatus); + task.updatePersistentStatus(newStatus, new ActionListener>() { + @Override + public void onResponse(PersistentTask persistentTask) { + logger.info("updating was successful"); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.info("updating failed", e); + latch.countDown(); + fail(e.toString()); + } + }); + assertTrue(latch.await(10, TimeUnit.SECONDS)); + } else if (testTask.isCancelled()) { + // Cancellation make cause different ways for the task to finish + if (randomBoolean()) { + if (randomBoolean()) { + task.markAsFailed(new TaskCancelledException(testTask.getReasonCancelled())); + } else { + task.markAsCompleted(); + } + } else { + task.markAsFailed(new RuntimeException(testTask.getReasonCancelled())); + } + return; + } else { + fail("We really shouldn't be here"); + } + } + } catch (InterruptedException e) { + task.markAsFailed(e); + } + } + + @Override + protected AllocatedPersistentTask createTask(long id, String type, String action, TaskId parentTaskId, + PersistentTask task, Map headers) { + return new TestTask(id, type, action, getDescription(task), parentTaskId, headers); + } + } + + public static class TestTaskAction extends Action { + + public static final TestTaskAction INSTANCE = new TestTaskAction(); + public static final String NAME = "cluster:admin/persistent/task_test"; + + private TestTaskAction() { + super(NAME); + } + + @Override + public TestTasksResponse newResponse() { + return new TestTasksResponse(); + } + + @Override + public TestTasksRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new TestTasksRequestBuilder(client); + } + } + + + public static class TestTask extends AllocatedPersistentTask { + private volatile String operation; + + public TestTask(long id, String type, String action, String description, TaskId parentTask, Map headers) { + super(id, type, action, description, parentTask, headers); + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + @Override + public String toString() { + return "TestTask[" + this.getId() + ", " + this.getParentTaskId() + ", " + this.getOperation() + "]"; + } + } + + static class TestTaskResponse implements Writeable { + + TestTaskResponse() { + + } + + TestTaskResponse(StreamInput in) throws IOException { + in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(true); + } + } + + public static class TestTasksRequest extends BaseTasksRequest { + private String operation; + + public TestTasksRequest() { + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + operation = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(operation); + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getOperation() { + return operation; + } + + } + + public static class TestTasksRequestBuilder extends TasksRequestBuilder { + + protected TestTasksRequestBuilder(ElasticsearchClient client) { + super(client, TestTaskAction.INSTANCE, new TestTasksRequest()); + } + + public TestTasksRequestBuilder setOperation(String operation) { + request.setOperation(operation); + return this; + } + } + + public static class TestTasksResponse extends BaseTasksResponse { + + private List tasks; + + public TestTasksResponse() { + super(null, null); + } + + public TestTasksResponse(List tasks, List taskFailures, + List nodeFailures) { + super(taskFailures, nodeFailures); + this.tasks = tasks == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(tasks)); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + tasks = in.readList(TestTaskResponse::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(tasks); + } + + public List getTasks() { + return tasks; + } + } + + public static class TransportTestTaskAction extends TransportTasksAction { + + @Inject + public TransportTestTaskAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, + TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, String nodeExecutor) { + super(settings, TestTaskAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, + TestTasksRequest::new, TestTasksResponse::new, ThreadPool.Names.MANAGEMENT); + } + + @Override + protected TestTasksResponse newResponse(TestTasksRequest request, List tasks, + List taskOperationFailures, + List failedNodeExceptions) { + return new TestTasksResponse(tasks, taskOperationFailures, failedNodeExceptions); + } + + @Override + protected TestTaskResponse readTaskResponse(StreamInput in) throws IOException { + return new TestTaskResponse(in); + } + + @Override + protected void taskOperation(TestTasksRequest request, TestTask task, ActionListener listener) { + task.setOperation(request.operation); + listener.onResponse(new TestTaskResponse()); + } + + } + + +} diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java new file mode 100644 index 0000000000000..6e20bb0009732 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.persistent; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.Status; +import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; +import org.elasticsearch.persistent.UpdatePersistentTaskStatusAction.Request; + +import java.util.Collections; + +public class UpdatePersistentTaskRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(UUIDs.base64UUID(), randomLong(), new Status(randomAlphaOfLength(10))); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.singletonList( + new NamedWriteableRegistry.Entry(Task.Status.class, TestPersistentTasksExecutor.NAME, Status::new) + )); + } +}