diff --git a/api/src/main/java/com/epam/pipeline/common/MessageConstants.java b/api/src/main/java/com/epam/pipeline/common/MessageConstants.java index a4182babc0..c36b595ee2 100644 --- a/api/src/main/java/com/epam/pipeline/common/MessageConstants.java +++ b/api/src/main/java/com/epam/pipeline/common/MessageConstants.java @@ -821,6 +821,11 @@ public final class MessageConstants { // Launch limits public static final String ERROR_RUN_LAUNCH_USER_LIMIT_EXCEEDED = "error.run.launch.user.limit.exceeded"; + public static final String ERROR_RUN_ASSIGN_POLICY_MALFORMED = "error.run.assign.policy.malformed"; + public static final String ERROR_RUN_ASSIGN_POLICY_FORBIDDEN = "error.run.assign.policy.forbidden"; + public static final String ERROR_RUN_WITH_SERVICE_ACCOUNT_FORBIDDEN = "error.run.with.service.account.forbidden"; + + // Ngs preprocessing public static final String ERROR_NGS_PREPROCESSING_FOLDER_ID_NOT_PROVIDED = diff --git a/api/src/main/java/com/epam/pipeline/controller/pipeline/issue/GitlabIssueController.java b/api/src/main/java/com/epam/pipeline/controller/pipeline/issue/GitlabIssueController.java index 0e3ec0d8a2..6b91232f42 100644 --- a/api/src/main/java/com/epam/pipeline/controller/pipeline/issue/GitlabIssueController.java +++ b/api/src/main/java/com/epam/pipeline/controller/pipeline/issue/GitlabIssueController.java @@ -90,7 +90,7 @@ public Result deleteIssue(@PathVariable(value = ISSUE_ID) final Long is return Result.success(gitlabIssueApiService.deleteIssue(issueId)); } - @GetMapping + @PostMapping(value = "/filter") @ApiOperation( value = "Gets all users issues.", notes = "Gets all users issues.", diff --git a/api/src/main/java/com/epam/pipeline/entity/billing/BillingGrouping.java b/api/src/main/java/com/epam/pipeline/entity/billing/BillingGrouping.java index 92a0bac2b9..2cc222fc17 100644 --- a/api/src/main/java/com/epam/pipeline/entity/billing/BillingGrouping.java +++ b/api/src/main/java/com/epam/pipeline/entity/billing/BillingGrouping.java @@ -19,6 +19,11 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + @Getter @RequiredArgsConstructor public enum BillingGrouping { @@ -36,4 +41,11 @@ public enum BillingGrouping { private final boolean runUsageDetailsRequired; private final boolean storageUsageDetailsRequired; + public static Set getStorageGrouping() { + return Collections.singleton(STORAGE); + } + + public static Set getRunGrouping() { + return new HashSet<>(Arrays.asList(RUN_COMPUTE_TYPE, PIPELINE, TOOL, USER, BILLING_CENTER)); + } } diff --git a/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingOrderAggregate.java b/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingOrderAggregate.java index 774ac623cf..7de78ad654 100644 --- a/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingOrderAggregate.java +++ b/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingOrderAggregate.java @@ -16,26 +16,36 @@ package com.epam.pipeline.entity.billing; +import com.epam.pipeline.manager.billing.BillingUtils; import lombok.Getter; import lombok.RequiredArgsConstructor; +import java.util.Set; + @Getter @RequiredArgsConstructor public enum BillingGroupingOrderAggregate { - DEFAULT(null, "cost", "cost"), + DEFAULT(null, BillingUtils.COST_FIELD, BillingUtils.COST_FIELD, null), - STORAGE(BillingGrouping.STORAGE, "cost", "usage_bytes"), - STANDARD(BillingGrouping.STORAGE, - "standard_total_cost", "standard_total_usage_bytes"), - GLACIER(BillingGrouping.STORAGE, - "glacier_total_cost", "glacier_total_usage_bytes"), - GLACIER_IR(BillingGrouping.STORAGE, - "glacier_ir_total_cost", "glacier_ir_total_usage_bytes"), - DEEP_ARCHIVE(BillingGrouping.STORAGE, - "deep_archive_total_cost", "deep_archive_total_usage_bytes"); + STORAGE(BillingGrouping.getStorageGrouping(), BillingUtils.COST_FIELD, BillingUtils.STORAGE_USAGE_FIELD, null), + STANDARD(BillingGrouping.getStorageGrouping(), + "standard_total_cost", "standard_total_usage_bytes", null), + GLACIER(BillingGrouping.getStorageGrouping(), + "glacier_total_cost", "glacier_total_usage_bytes", null), + GLACIER_IR(BillingGrouping.getStorageGrouping(), + "glacier_ir_total_cost", "glacier_ir_total_usage_bytes", null), + DEEP_ARCHIVE(BillingGrouping.getStorageGrouping(), + "deep_archive_total_cost", "deep_archive_total_usage_bytes", null), + RUN(BillingGrouping.getRunGrouping(), BillingUtils.COST_FIELD, + BillingUtils.RUN_USAGE_FIELD, BillingUtils.RUN_ID_FIELD), + RUN_COMPUTE(BillingGrouping.getRunGrouping(), BillingUtils.COMPUTE_COST_FIELD, + BillingUtils.RUN_USAGE_FIELD, BillingUtils.RUN_ID_FIELD), + RUN_DISK(BillingGrouping.getRunGrouping(), BillingUtils.DISK_COST_FIELD, + BillingUtils.RUN_USAGE_FIELD, BillingUtils.RUN_ID_FIELD); - private final BillingGrouping group; + private final Set groups; private final String costField; private final String usageField; + private final String countField; } diff --git a/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingSortOrder.java b/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingSortOrder.java index 21845cca6b..7877e14c46 100644 --- a/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingSortOrder.java +++ b/api/src/main/java/com/epam/pipeline/entity/billing/BillingGroupingSortOrder.java @@ -16,9 +16,12 @@ package com.epam.pipeline.entity.billing; +import com.epam.pipeline.manager.billing.BillingUtils; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; @Getter @RequiredArgsConstructor @@ -32,7 +35,7 @@ public class BillingGroupingSortOrder { ); public enum BillingGroupingSortMetric { - COST, USAGE + COST, USAGE, USAGE_RUNS, COUNT_RUNS } private final BillingGroupingSortMetric metric; @@ -44,11 +47,29 @@ public String getAggregateToOrderBy() { case COST: return aggregate.getCostField(); case USAGE: + case USAGE_RUNS: return aggregate.getUsageField(); + case COUNT_RUNS: + return aggregate.getCountField(); default: return BillingGroupingOrderAggregate.DEFAULT.getCostField(); } } + + public AggregationBuilder getAggregation() { + final String aggregateField = getAggregateToOrderBy(); + switch (metric) { + case USAGE: + return AggregationBuilders.avg(aggregateField + BillingUtils.SORT_AGG_POSTFIX) + .field(aggregateField); + case COUNT_RUNS: + return AggregationBuilders.count(aggregateField + BillingUtils.SORT_AGG_POSTFIX) + .field(aggregateField); + default: + return AggregationBuilders.sum(aggregateField + BillingUtils.SORT_AGG_POSTFIX) + .field(aggregateField); + } + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/billing/BillingUtils.java b/api/src/main/java/com/epam/pipeline/manager/billing/BillingUtils.java index 9f54260739..b446846be3 100644 --- a/api/src/main/java/com/epam/pipeline/manager/billing/BillingUtils.java +++ b/api/src/main/java/com/epam/pipeline/manager/billing/BillingUtils.java @@ -152,6 +152,7 @@ public final class BillingUtils { public static final String DISCOUNT_SCRIPT_TEMPLATE = "_value + _value * (%s)"; public static final String RESOURCE_TYPE = "resource_type"; public static final String COMPUTE_GROUP = "COMPUTE"; + public static final String SORT_AGG_POSTFIX = "_sort_order"; private BillingUtils() { } diff --git a/api/src/main/java/com/epam/pipeline/manager/billing/order/BillingOrderApplier.java b/api/src/main/java/com/epam/pipeline/manager/billing/order/BillingOrderApplier.java index f2797ae0a3..b7bb8d7130 100644 --- a/api/src/main/java/com/epam/pipeline/manager/billing/order/BillingOrderApplier.java +++ b/api/src/main/java/com/epam/pipeline/manager/billing/order/BillingOrderApplier.java @@ -19,17 +19,15 @@ import com.epam.pipeline.entity.billing.BillingGrouping; import com.epam.pipeline.entity.billing.BillingGroupingOrderAggregate; import com.epam.pipeline.entity.billing.BillingGroupingSortOrder; +import com.epam.pipeline.manager.billing.BillingUtils; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.springframework.util.Assert; public final class BillingOrderApplier { - public static final String SORT_AGG_POSTFIX = "_sort_order"; - private BillingOrderApplier() {} public static BoolQueryBuilder applyOrder(final BillingGrouping grouping, @@ -37,41 +35,17 @@ public static BoolQueryBuilder applyOrder(final BillingGrouping grouping, final BoolQueryBuilder query, final TermsAggregationBuilder terms) { final BillingGroupingOrderAggregate orderAggregate = order.getAggregate(); - Assert.isTrue(orderAggregate.getGroup() == null || grouping.equals(orderAggregate.getGroup()), + Assert.isTrue(orderAggregate.getGroups() == null || orderAggregate.getGroups().contains(grouping), String.format("Grouping: %s and Grouping Order: %s, don't match.", grouping.name(), orderAggregate.name())); // Apply additional filter to query to filter out docs that don't have value to sort by query.filter(QueryBuilders.boolQuery().must(QueryBuilders.existsQuery(order.getAggregateToOrderBy()))); - // Depends on Grouping type apply specific logic for ordering results - if (grouping == BillingGrouping.STORAGE) { - storageGroupingOrder(order, terms); - } else { - defaultOrder(order, terms); - } - return query; - } + terms.subAggregation(order.getAggregation()); + terms.order(BucketOrder.aggregation(order.getAggregateToOrderBy() + BillingUtils.SORT_AGG_POSTFIX, + order.isDesc())); - private static void storageGroupingOrder(final BillingGroupingSortOrder order, - final TermsAggregationBuilder terms) { - final String aggToOrderBy = order.getAggregateToOrderBy(); - if (order.getMetric() == BillingGroupingSortOrder.BillingGroupingSortMetric.COST) { - terms.subAggregation(AggregationBuilders.sum(aggToOrderBy + SORT_AGG_POSTFIX).field(aggToOrderBy)); - } else { - terms.subAggregation(AggregationBuilders.avg(aggToOrderBy + SORT_AGG_POSTFIX).field(aggToOrderBy)); - } - terms.order(BucketOrder.aggregation(aggToOrderBy + SORT_AGG_POSTFIX, order.isDesc())); - } - - private static void defaultOrder(final BillingGroupingSortOrder order, final TermsAggregationBuilder terms) { - terms.subAggregation( - AggregationBuilders.sum(order.getAggregateToOrderBy() + SORT_AGG_POSTFIX) - .field(order.getAggregateToOrderBy()) - ); - terms.order( - BucketOrder.aggregation(order.getAggregateToOrderBy() + SORT_AGG_POSTFIX, order.isDesc()) - ); + return query; } - } diff --git a/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java b/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java index 35ce11a133..7be189cafb 100644 --- a/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java @@ -29,6 +29,7 @@ import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.ToolGroup; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.entity.utils.DateUtils; import com.epam.pipeline.exception.CmdExecutionException; @@ -331,8 +332,7 @@ private void launchPodIfRequired(final PipelineRun run, final List endpo try { if (Objects.isNull(kubernetesManager.findPodById(run.getPodId()))) { final PipelineConfiguration configuration = getResumeConfiguration(run); - launcher.launch(run, configuration, endpoints, run.getId().toString(), - true, run.getPodId(), null, ImagePullPolicy.NEVER); + launcher.launch(run, configuration, endpoints, true, run.getPodId(), null, ImagePullPolicy.NEVER); } } finally { resumeLock.unlock(); @@ -343,6 +343,14 @@ private PipelineConfiguration getResumeConfiguration(final PipelineRun run) { final PipelineConfiguration configuration = configurationManager.getConfigurationFromRun(run); final Map envs = getResumeRunEnvVars(configuration); configuration.setEnvironmentParams(envs); + configuration.setPodAssignPolicy( + RunAssignPolicy.builder() + .selector( + RunAssignPolicy.PodAssignSelector.builder() + .label(KubernetesConstants.RUN_ID_LABEL) + .value(run.getId().toString()).build()) + .build() + ); return configuration; } diff --git a/api/src/main/java/com/epam/pipeline/manager/execution/PipelineExecutor.java b/api/src/main/java/com/epam/pipeline/manager/execution/PipelineExecutor.java index acad5e728f..dd5df2f46d 100644 --- a/api/src/main/java/com/epam/pipeline/manager/execution/PipelineExecutor.java +++ b/api/src/main/java/com/epam/pipeline/manager/execution/PipelineExecutor.java @@ -20,6 +20,7 @@ import com.epam.pipeline.entity.cluster.container.ContainerMemoryResourcePolicy; import com.epam.pipeline.entity.cluster.container.ImagePullPolicy; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; import com.epam.pipeline.manager.cluster.KubernetesConstants; import com.epam.pipeline.manager.cluster.KubernetesManager; import com.epam.pipeline.manager.cluster.container.ContainerMemoryResourceService; @@ -87,6 +88,7 @@ public class PipelineExecutor { .hostPath("/sys/fs/cgroup") .mountPath("/sys/fs/cgroup").build(); private static final String DOMAIN_DELIMITER = "@"; + private static final String DEFAULT_KUBE_SERVICE_ACCOUNT = "default"; private final PreferenceManager preferenceManager; private final String kubeNamespace; @@ -107,17 +109,20 @@ public PipelineExecutor(final PreferenceManager preferenceManager, this.kubernetesManager = kubernetesManager; } - public void launchRootPod(String command, PipelineRun run, List envVars, List endpoints, - String pipelineId, String nodeIdLabel, String secretName, String clusterId) { - launchRootPod(command, run, envVars, endpoints, pipelineId, nodeIdLabel, secretName, clusterId, - ImagePullPolicy.ALWAYS, Collections.emptyMap()); + public void launchRootPod(final String command, final PipelineRun run, final List envVars, + final List endpoints, final String pipelineId, + final RunAssignPolicy podAssignPolicy, final String secretName, final String clusterId) { + launchRootPod(command, run, envVars, endpoints, pipelineId, podAssignPolicy, + secretName, clusterId, ImagePullPolicy.ALWAYS, Collections.emptyMap(), null); } - public void launchRootPod(String command, PipelineRun run, List envVars, List endpoints, - String pipelineId, String nodeIdLabel, String secretName, String clusterId, - ImagePullPolicy imagePullPolicy, Map kubeLabels) { + public void launchRootPod(final String command, final PipelineRun run, final List envVars, + final List endpoints, final String pipelineId, + final RunAssignPolicy podAssignPolicy, final String secretName, final String clusterId, + final ImagePullPolicy imagePullPolicy, Map kubeLabels, + final String kubeServiceAccount) { try (KubernetesClient client = kubernetesManager.getKubernetesClient()) { - Map labels = new HashMap<>(); + final Map labels = new HashMap<>(); labels.put("spawned_by", "pipeline-api"); labels.put("pipeline_id", pipelineId); labels.put("owner", normalizeOwner(run.getOwner())); @@ -129,13 +134,14 @@ public void launchRootPod(String command, PipelineRun run, List envVars, } addWorkerLabel(clusterId, labels, run); LOGGER.debug("Root pipeline task ID: {}", run.getPodId()); - Map nodeSelector = new HashMap<>(); - String runIdLabel = String.valueOf(run.getId()); + final Map nodeSelector = new HashMap<>(); + final String runIdLabel = String.valueOf(run.getId()); if (preferenceManager.getPreference(SystemPreferences.CLUSTER_ENABLE_AUTOSCALING)) { - nodeSelector.put(KubernetesConstants.RUN_ID_LABEL, nodeIdLabel); + nodeSelector.put(podAssignPolicy.getSelector().getLabel(), podAssignPolicy.getSelector().getValue()); // id pod ip == pipeline id we have a root pod, otherwise we prefer to skip pod in autoscaler - if (run.getPodId().equals(pipelineId) && nodeIdLabel.equals(runIdLabel)) { + if (run.getPodId().equals(pipelineId) && + podAssignPolicy.isMatch(KubernetesConstants.RUN_ID_LABEL, runIdLabel)) { labels.put(KubernetesConstants.TYPE_LABEL, KubernetesConstants.PIPELINE_TYPE); } labels.put(KubernetesConstants.RUN_ID_LABEL, runIdLabel); @@ -145,12 +151,15 @@ public void launchRootPod(String command, PipelineRun run, List envVars, labels.putAll(getServiceLabels(endpoints)); - OkHttpClient httpClient = HttpClientUtils.createHttpClient(client.getConfiguration()); - ObjectMeta metadata = getObjectMeta(run, labels); - PodSpec spec = getPodSpec(run, envVars, secretName, nodeSelector, run.getActualDockerImage(), command, - imagePullPolicy, nodeIdLabel.equals(runIdLabel)); - Pod pod = new Pod("v1", "Pod", metadata, spec, null); - Pod created = new PodOperationsImpl(httpClient, client.getConfiguration(), kubeNamespace).create(pod); + final OkHttpClient httpClient = HttpClientUtils.createHttpClient(client.getConfiguration()); + final ObjectMeta metadata = getObjectMeta(run, labels); + final String verifiedKubeServiceAccount = fetchVerifiedKubeServiceAccount(client, kubeServiceAccount); + final PodSpec spec = getPodSpec(run, envVars, secretName, nodeSelector, podAssignPolicy.getTolerances(), + run.getActualDockerImage(), command, imagePullPolicy, + podAssignPolicy.isMatch(KubernetesConstants.RUN_ID_LABEL, runIdLabel), + verifiedKubeServiceAccount); + final Pod pod = new Pod("v1", "Pod", metadata, spec, null); + final Pod created = new PodOperationsImpl(httpClient, client.getConfiguration(), kubeNamespace).create(pod); LOGGER.debug("Created POD: {}", created.toString()); } } @@ -159,6 +168,24 @@ private String normalizeOwner(final String owner) { return splitName(owner).replaceAll(KubernetesConstants.KUBE_NAME_FULL_REGEXP, "-"); } + private String fetchVerifiedKubeServiceAccount(final KubernetesClient client, final String kubeServiceAccount) { + if (StringUtils.isNotBlank(kubeServiceAccount)) { + return client.serviceAccounts() + .inNamespace(kubeNamespace).list().getItems().stream() + .filter(serviceAccount -> serviceAccount.getMetadata().getName().equals(kubeServiceAccount)) + .map(serviceAccount -> serviceAccount.getMetadata().getName()) + .findFirst() + .orElseGet(() -> { + LOGGER.warn(String.format( + "Can't find kube service account that was requested: %s! Default will be used", + kubeServiceAccount)); + return DEFAULT_KUBE_SERVICE_ACCOUNT; + }); + } else { + return DEFAULT_KUBE_SERVICE_ACCOUNT; + } + } + private String splitName(final String owner) { return owner.split(DOMAIN_DELIMITER)[0]; } @@ -175,10 +202,11 @@ private String getChildLabel(final String clusterId, final PipelineRun run) { Optional.ofNullable(run.getParentRunId()).map(String::valueOf).orElse(StringUtils.EMPTY)); } - private PodSpec getPodSpec(PipelineRun run, List envVars, String secretName, - Map nodeSelector, String dockerImage, - String command, ImagePullPolicy imagePullPolicy, boolean isParentPod) { - PodSpec spec = new PodSpec(); + private PodSpec getPodSpec(final PipelineRun run, final List envVars, final String secretName, + final Map nodeSelector, final Map nodeTolerances, + final String dockerImage, final String command, final ImagePullPolicy imagePullPolicy, + final boolean isParentPod, final String kubeServiceAccount) { + final PodSpec spec = new PodSpec(); spec.setRestartPolicy("Never"); spec.setTerminationGracePeriodSeconds(KUBE_TERMINATION_PERIOD); spec.setDnsPolicy("ClusterFirst"); @@ -194,9 +222,11 @@ private PodSpec getPodSpec(PipelineRun run, List envVars, String secretN if (!StringUtils.isEmpty(secretName)) { spec.setImagePullSecrets(Collections.singletonList(new LocalObjectReference(secretName))); } - boolean isDockerInDockerEnabled = authManager.isAdmin() && isParameterEnabled(envVars, + final boolean isDockerInDockerEnabled = authManager.isAdmin() && isParameterEnabled(envVars, KubernetesConstants.CP_CAP_DIND_NATIVE); - boolean isSystemdEnabled = isParameterEnabled(envVars, KubernetesConstants.CP_CAP_SYSTEMD_CONTAINER); + final boolean isSystemdEnabled = isParameterEnabled(envVars, KubernetesConstants.CP_CAP_SYSTEMD_CONTAINER); + + spec.setServiceAccountName(kubeServiceAccount); if (KubernetesConstants.WINDOWS.equals(run.getPlatform())) { spec.setVolumes(getWindowsVolumes()); @@ -204,6 +234,10 @@ private PodSpec getPodSpec(PipelineRun run, List envVars, String secretN spec.setVolumes(getVolumes(isDockerInDockerEnabled, isSystemdEnabled)); } + Optional.of(PodSpecMapperHelper.buildTolerations(nodeTolerances)) + .filter(CollectionUtils::isNotEmpty) + .ifPresent(spec::setTolerations); + if (envVars.stream().anyMatch(envVar -> envVar.getName().equals(USE_HOST_NETWORK))){ spec.setHostNetwork(true); } diff --git a/api/src/main/java/com/epam/pipeline/manager/execution/PipelineLauncher.java b/api/src/main/java/com/epam/pipeline/manager/execution/PipelineLauncher.java index e60f20623a..207cae7448 100644 --- a/api/src/main/java/com/epam/pipeline/manager/execution/PipelineLauncher.java +++ b/api/src/main/java/com/epam/pipeline/manager/execution/PipelineLauncher.java @@ -25,7 +25,9 @@ import com.epam.pipeline.entity.configuration.PipelineConfiguration; import com.epam.pipeline.entity.git.GitCredentials; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; import com.epam.pipeline.entity.pipeline.run.parameter.RunSid; +import com.epam.pipeline.entity.user.PipelineUser; import com.epam.pipeline.manager.cloud.CloudFacade; import com.epam.pipeline.manager.cluster.KubernetesConstants; import com.epam.pipeline.manager.preference.PreferenceManager; @@ -103,40 +105,40 @@ public class PipelineLauncher { private final SimpleDateFormat dateFormat = new SimpleDateFormat(Constants.SIMPLE_DATE_FORMAT); private final SimpleDateFormat timeFormat = new SimpleDateFormat(Constants.SIMPLE_TIME_FORMAT); - public String launch(PipelineRun run, PipelineConfiguration configuration, List endpoints, - String nodeIdLabel, String clusterId) { - return launch(run, configuration, endpoints, nodeIdLabel, true, run.getPodId(), clusterId); + public String launch(final PipelineRun run, final PipelineConfiguration configuration, + final List endpoints, final String clusterId) { + return launch(run, configuration, endpoints, true, run.getPodId(), clusterId); } - public String launch(PipelineRun run, PipelineConfiguration configuration, - List endpoints, String nodeIdLabel, boolean useLaunch, - String pipelineId, String clusterId) { - return launch(run, configuration, endpoints, nodeIdLabel, useLaunch, pipelineId, clusterId, - getImagePullPolicy(configuration)); + public String launch(final PipelineRun run, final PipelineConfiguration configuration, final List endpoints, + final boolean useLaunch, final String pipelineId, final String clusterId) { + return launch(run, configuration, endpoints, useLaunch, pipelineId, + clusterId, getImagePullPolicy(configuration)); } - public String launch(PipelineRun run, PipelineConfiguration configuration, - List endpoints, String nodeIdLabel, boolean useLaunch, - String pipelineId, String clusterId, ImagePullPolicy imagePullPolicy) { - GitCredentials gitCredentials = configuration.getGitCredentials(); + public String launch(final PipelineRun run, final PipelineConfiguration configuration, + final List endpoints, final boolean useLaunch, final String pipelineId, + final String clusterId, final ImagePullPolicy imagePullPolicy) { + validateLaunchConfiguration(configuration); + final GitCredentials gitCredentials = configuration.getGitCredentials(); //TODO: AZURE fix - Map systemParams = matchSystemParams( + final Map systemParams = matchSystemParams( run, preferenceManager.getPreference(SystemPreferences.BASE_API_HOST), kubeNamespace, preferenceManager.getPreference(SystemPreferences.CLUSTER_ENABLE_AUTOSCALING), configuration, gitCredentials); - checkRunOnParentNode(run, nodeIdLabel, systemParams); - List envVars = EnvVarsBuilder.buildEnvVars(run, configuration, systemParams, + markRunOnParentNode(run, configuration.getPodAssignPolicy(), systemParams); + final List envVars = EnvVarsBuilder.buildEnvVars(run, configuration, systemParams, buildRegionSpecificEnvVars(run.getInstance().getCloudRegionId(), run.getSensitive(), configuration.getKubeLabels())); Assert.isTrue(!StringUtils.isEmpty(configuration.getCmdTemplate()), messageHelper.getMessage( MessageConstants.ERROR_CMD_TEMPLATE_NOT_RESOLVED)); - String pipelineCommand = commandBuilder.build(configuration, systemParams); - String gitCloneUrl = Optional.ofNullable(gitCredentials).map(GitCredentials::getUrl) + final String pipelineCommand = commandBuilder.build(configuration, systemParams); + final String gitCloneUrl = Optional.ofNullable(gitCredentials).map(GitCredentials::getUrl) .orElse(run.getRepository()); - String rootPodCommand; + final String rootPodCommand; if (!useLaunch) { rootPodCommand = pipelineCommand; } else { @@ -152,12 +154,54 @@ public String launch(PipelineRun run, PipelineConfiguration configuration, } } LOGGER.debug("Start script command: {}", rootPodCommand); - executor.launchRootPod(rootPodCommand, run, envVars, - endpoints, pipelineId, nodeIdLabel, configuration.getSecretName(), - clusterId, imagePullPolicy, configuration.getKubeLabels()); + executor.launchRootPod(rootPodCommand, run, envVars, endpoints, pipelineId, + configuration.getPodAssignPolicy(), configuration.getSecretName(), + clusterId, imagePullPolicy, configuration.getKubeLabels(), + configuration.getKubeServiceAccount()); return pipelineCommand; } + void validateLaunchConfiguration(final PipelineConfiguration configuration) { + final PipelineUser user = authManager.getCurrentUser(); + validateRunAssignPolicy(configuration); + validateConfigurationOnAdvancedAssignPolicy(configuration, user); + validateConfigurationOnKubernetesServiceAccount(configuration, user); + } + + private void validateRunAssignPolicy(final PipelineConfiguration configuration) { + Optional.ofNullable(configuration.getPodAssignPolicy()).ifPresent(assignPolicy -> { + if (!assignPolicy.isValid()) { + throw new IllegalArgumentException( + messageHelper.getMessage(MessageConstants.ERROR_RUN_ASSIGN_POLICY_MALFORMED, + assignPolicy.toString())); + } + }); + } + + private void validateConfigurationOnKubernetesServiceAccount(final PipelineConfiguration configuration, + final PipelineUser user) { + if (!user.isAdmin() && configuration.getKubeServiceAccount() != null) { + throw new IllegalStateException( + messageHelper.getMessage( + MessageConstants.ERROR_RUN_WITH_SERVICE_ACCOUNT_FORBIDDEN, user.getUserName()) + ); + } + } + + private void validateConfigurationOnAdvancedAssignPolicy(final PipelineConfiguration configuration, + final PipelineUser user) { + if (user.isAdmin()) { + return; + } + final boolean isAdvancedRunAssignPolicy = Optional.ofNullable(configuration.getPodAssignPolicy()) + .map(policy -> !policy.getSelector().getLabel().equals(KubernetesConstants.RUN_ID_LABEL)) + .orElse(false); + if (isAdvancedRunAssignPolicy) { + throw new IllegalStateException( + messageHelper.getMessage(MessageConstants.ERROR_RUN_ASSIGN_POLICY_FORBIDDEN, user.getUserName())); + } + } + private Map buildRegionSpecificEnvVars(final Long cloudRegionId, final boolean sensitiveRun, final Map kubeLabels) { @@ -204,10 +248,15 @@ private Map getExternalProperties(final Long regionId, return new ObjectMapper().convertValue(mergedEnvVars, new TypeReference>() {}); } - private void checkRunOnParentNode(PipelineRun run, String nodeIdLabel, - Map systemParams) { - if (!run.getId().toString().equals(nodeIdLabel)) { - systemParams.put(SystemParams.RUN_ON_PARENT_NODE, EMPTY_PARAMETER); + private void markRunOnParentNode(final PipelineRun run, final RunAssignPolicy assignPolicy, + final Map systemParams) { + if (assignPolicy != null && assignPolicy.isValid()) { + assignPolicy.ifMatchThenMapValue(KubernetesConstants.RUN_ID_LABEL, Long::valueOf) + .ifPresent(parentNodeId -> { + if (!run.getId().equals(parentNodeId)) { + systemParams.put(SystemParams.RUN_ON_PARENT_NODE, EMPTY_PARAMETER); + } + }); } } diff --git a/api/src/main/java/com/epam/pipeline/manager/execution/PodSpecMapperHelper.java b/api/src/main/java/com/epam/pipeline/manager/execution/PodSpecMapperHelper.java new file mode 100644 index 0000000000..ad4666a00f --- /dev/null +++ b/api/src/main/java/com/epam/pipeline/manager/execution/PodSpecMapperHelper.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.manager.execution; + +import io.fabric8.kubernetes.api.model.Toleration; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public final class PodSpecMapperHelper { + + private PodSpecMapperHelper() {} + + public static final String TOLERATION_OP_EQUAL = "Equal"; + public static final String TOLERATION_OP_EXISTS = "Exists"; + + public static List buildTolerations(final Map nodeTolerances) { + return MapUtils.emptyIfNull(nodeTolerances) + .entrySet() + .stream().map(t -> { + final Toleration toleration = new Toleration(); + toleration.setKey(t.getKey()); + if (StringUtils.isNotBlank(t.getValue())) { + toleration.setValue(t.getValue()); + toleration.setOperator(TOLERATION_OP_EQUAL); + } else { + toleration.setOperator(TOLERATION_OP_EXISTS); + } + return toleration; + }).collect(Collectors.toList()); + } +} diff --git a/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManager.java b/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManager.java index b566523a6f..70c8cb8bb9 100644 --- a/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManager.java @@ -30,9 +30,11 @@ import com.epam.pipeline.entity.pipeline.RunInstance; import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.entity.utils.DefaultSystemParameter; import com.epam.pipeline.exception.git.GitClientException; +import com.epam.pipeline.manager.cluster.KubernetesConstants; import com.epam.pipeline.manager.docker.ToolVersionManager; import com.epam.pipeline.manager.git.GitManager; import com.epam.pipeline.manager.preference.PreferenceManager; @@ -208,6 +210,16 @@ public PipelineConfiguration mergeParameters(PipelineStart runVO, PipelineConfig configuration.setKubeLabels(defaultConfig.getKubeLabels()); } + // TODO: merging parentNodeId together with runAssignPolicy, + // in a future we can delete it if we get rid of parentNodeId in favor of runAssignPolicy + configuration.setPodAssignPolicy(mergeAssignPolicy(runVO, defaultConfig)); + + if (!StringUtils.isEmpty(runVO.getKubeServiceAccount())) { + configuration.setKubeServiceAccount(runVO.getKubeServiceAccount()); + } else { + configuration.setKubeServiceAccount(defaultConfig.getKubeServiceAccount()); + } + //client always sends actual node-count configuration.setNodeCount(runVO.getNodeCount()); @@ -232,6 +244,42 @@ public PipelineConfiguration mergeParameters(PipelineStart runVO, PipelineConfig return configuration; } + private RunAssignPolicy mergeAssignPolicy(final PipelineStart runVO, final PipelineConfiguration defaultConfig) { + final Long useRunId = runVO.getParentNodeId() != null ? runVO.getParentNodeId() : runVO.getUseRunId(); + final RunAssignPolicy assignPolicy = runVO.getPodAssignPolicy(); + + if (useRunId != null && assignPolicy != null) { + throw new IllegalArgumentException( + "Both RunAssignPolicy and (parentRunId or useRunId) cannot be specified, " + + "please provide only one of them" + ); + } + + if (assignPolicy != null && assignPolicy.isValid()) { + log.debug("RunAssignPolicy is provided and valid, will proceed with it."); + return assignPolicy; + } else { + if (useRunId != null) { + final String value = useRunId.toString(); + log.debug( + String.format("Configuring RunAssignPolicy as: label %s, value: %s.", + KubernetesConstants.RUN_ID_LABEL, value) + ); + return RunAssignPolicy.builder() + .selector( + RunAssignPolicy.PodAssignSelector.builder() + .label(KubernetesConstants.RUN_ID_LABEL) + .value(value).build()) + .build(); + } else { + if (defaultConfig.getPodAssignPolicy() != null && defaultConfig.getPodAssignPolicy().isValid()) { + return defaultConfig.getPodAssignPolicy(); + } + return RunAssignPolicy.builder().build(); + } + } + } + /** * Modifies configuration if this a cluster run configuration. * @@ -260,11 +308,20 @@ public void updateMasterConfiguration(PipelineConfiguration configuration, boole configuration.buildEnvVariables(); } - public void updateWorkerConfiguration(String parentId, PipelineStart runVO, - PipelineConfiguration configuration, boolean isNFS, boolean clearParams) { - configuration.setEraseRunEndpoints(hasBooleanParameter(configuration, ERASE_WORKER_ENDPOINTS)); - final Map configParameters = MapUtils.isEmpty(configuration.getParameters()) ? - new HashMap<>() : configuration.getParameters(); + public PipelineConfiguration generateMasterConfiguration(final PipelineConfiguration configuration, + final boolean isNFS) { + final PipelineConfiguration copiedConfiguration = configuration.clone(); + updateMasterConfiguration(copiedConfiguration, isNFS); + return copiedConfiguration; + } + + public PipelineConfiguration generateWorkerConfiguration(final String parentId, final PipelineStart runVO, + final PipelineConfiguration configuration, + final boolean isNFS, final boolean clearParams) { + final PipelineConfiguration workerConfiguration = configuration.clone(); + workerConfiguration.setEraseRunEndpoints(hasBooleanParameter(workerConfiguration, ERASE_WORKER_ENDPOINTS)); + final Map configParameters = MapUtils.isEmpty(workerConfiguration.getParameters()) ? + new HashMap<>() : workerConfiguration.getParameters(); final Map updatedParams = clearParams ? new HashMap<>() : configParameters; final List systemParameters = preferenceManager.getPreference( SystemPreferences.LAUNCH_SYSTEM_PARAMETERS); @@ -283,14 +340,21 @@ public void updateWorkerConfiguration(String parentId, PipelineStart runVO, } else { updatedParams.remove(NFS_CLUSTER_ROLE); } - configuration.setParameters(updatedParams); - configuration.setClusterRole(WORKER_CLUSTER_ROLE); - configuration.setCmdTemplate(StringUtils.hasText(runVO.getWorkerCmd()) ? + workerConfiguration.setParameters(updatedParams); + workerConfiguration.setClusterRole(WORKER_CLUSTER_ROLE); + workerConfiguration.setCmdTemplate(StringUtils.hasText(runVO.getWorkerCmd()) ? runVO.getWorkerCmd() : WORKER_CMD_TEMPLATE); - configuration.setPrettyUrl(null); + workerConfiguration.setPrettyUrl(null); //remove node count parameter for workers - configuration.setNodeCount(null); - configuration.buildEnvVariables(); + workerConfiguration.setNodeCount(null); + // if podAssignPolicy is a simple policy to assign run pod to dedicated instance, then we need to cleared it + // and workers then will be assigned to its own nodes, otherwise keep existing policy to assign workers + // as was configured in policy object + if (workerConfiguration.getPodAssignPolicy().isMatch(KubernetesConstants.RUN_ID_LABEL, parentId)) { + workerConfiguration.setPodAssignPolicy(null); + } + workerConfiguration.buildEnvVariables(); + return workerConfiguration; } public boolean hasNFSParameter(PipelineConfiguration entry) { diff --git a/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java b/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java index 067114cb52..64a735da78 100644 --- a/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java @@ -57,6 +57,7 @@ import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.pipeline.run.PipelineStartNotificationRequest; import com.epam.pipeline.entity.pipeline.run.RestartRun; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; import com.epam.pipeline.entity.pipeline.run.RunInfo; import com.epam.pipeline.entity.pipeline.run.RunStatus; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; @@ -258,7 +259,7 @@ public PipelineRun runCmd(final PipelineStart runVO) { final boolean clusterRun = configurationManager.initClusterConfiguration(configuration, true); final PipelineRun run = launchPipeline(configuration, null, null, - runVO.getInstanceType(), runVO.getParentNodeId(), runVO.getConfigurationName(), null, + runVO.getInstanceType(), runVO.getConfigurationName(), null, runVO.getParentRunId(), null, null, runVO.getRunSids(), configuration.getNotifications()); run.setParent(tool); @@ -278,18 +279,18 @@ public PipelineRun runCmd(final PipelineStart runVO) { //TODO: refactoring @ToolSecurityPolicyCheck @Transactional(propagation = Propagation.REQUIRED) - public PipelineRun runPod(PipelineStart runVO) { + public PipelineRun runPod(final PipelineStart runVO) { Assert.notNull(runVO.getCmdTemplate(), messageHelper.getMessage(MessageConstants.SETTING_IS_NOT_PROVIDED, "cmd_template")); - PipelineRun parentRun = loadPipelineRun(runVO.getUseRunId(), false); + final PipelineRun parentRun = loadPipelineRun(runVO.getUseRunId(), false); Assert.state(parentRun.getStatus() == TaskStatus.RUNNING, messageHelper.getMessage(MessageConstants.ERROR_PIPELINE_RUN_NOT_RUNNING, runVO.getUseRunId())); checkRunLaunchLimits(runVO); - PipelineConfiguration configuration = configurationManager.getPipelineConfiguration(runVO); - Tool tool = getToolForRun(configuration); + final PipelineConfiguration configuration = configurationManager.getPipelineConfiguration(runVO); + final Tool tool = getToolForRun(configuration); configuration.setSecretName(tool.getSecretName()); - List endpoints = tool.getEndpoints(); - PipelineRun run = new PipelineRun(); + final List endpoints = tool.getEndpoints(); + final PipelineRun run = new PipelineRun(); run.setInstance(parentRun.getInstance()); run.setId(runVO.getUseRunId()); run.setStartDate(DateUtils.now()); @@ -304,8 +305,10 @@ public PipelineRun runPod(PipelineStart runVO) { run.setLastChangeCommitTime(DateUtils.now()); run.setRunSids(runVO.getRunSids()); run.setOwner(parentRun.getOwner()); - String launchedCommand = pipelineLauncher.launch(run, configuration, - endpoints, runVO.getUseRunId().toString(), false, parentRun.getPodId(), null); + run.setSensitive(checkRunForSensitivity(MapUtils.emptyIfNull(configuration.getParameters()))); + final String launchedCommand = pipelineLauncher.launch( + run, configuration, endpoints, false, parentRun.getPodId(), null + ); run.setActualCmd(launchedCommand); return run; } @@ -335,7 +338,7 @@ public PipelineRun runPipeline(final PipelineStart runVO) { permissionManager.checkToolRunPermission(configuration.getDockerImage()); final PipelineRun run = launchPipeline(configuration, pipeline, version, - runVO.getInstanceType(), runVO.getParentNodeId(), runVO.getConfigurationName(), null, + runVO.getInstanceType(), runVO.getConfigurationName(), null, runVO.getParentRunId(), null, null, runVO.getRunSids(), configuration.getNotifications()); run.setParent(pipeline); @@ -368,18 +371,21 @@ public void prolongIdleRun(Long runId) { * @return */ @Transactional(propagation = Propagation.REQUIRED) - public PipelineRun launchPipeline(PipelineConfiguration configuration, Pipeline pipeline, String version, - String instanceType, Long parentNodeId, String configurationName, String clusterId, - Long parentRunId, List entityIds, Long configurationId, List runSids, - List notificationRequests) { - Optional parentRun = resolveParentRun(parentRunId, configuration); - Tool tool = getToolForRun(configuration); - Optional toolVersion = toolManager.findToolVersion(tool); - PipelineConfiguration toolConfiguration = configurationManager.getConfigurationForTool(tool, configuration); - AbstractCloudRegion region = resolveCloudRegion(parentRun.orElse(null), configuration, toolConfiguration); + public PipelineRun launchPipeline(final PipelineConfiguration configuration, final Pipeline pipeline, + final String version, final String instanceType, final String configurationName, + final String clusterId, final Long parentRunId, final List entityIds, + final Long configurationId, final List runSids, + final List notificationRequests) { + final Optional parentRun = resolveParentRun(parentRunId, configuration); + final Tool tool = getToolForRun(configuration); + final Optional toolVersion = toolManager.findToolVersion(tool); + final PipelineConfiguration toolConfiguration = configurationManager + .getConfigurationForTool(tool, configuration); + final AbstractCloudRegion region = resolveCloudRegion( + parentRun.orElse(null), configuration, toolConfiguration); validateCloudRegion(toolConfiguration, region); validateInstanceAndPriceTypes(configuration, pipeline, region, instanceType); - String instanceDisk = configuration.getInstanceDisk(); + final String instanceDisk = configuration.getInstanceDisk(); if (StringUtils.hasText(instanceDisk)) { Assert.isTrue(NumberUtils.isNumber(instanceDisk) && Integer.parseInt(instanceDisk) > 0, @@ -388,22 +394,41 @@ public PipelineRun launchPipeline(PipelineConfiguration configuration, Pipeline adjustInstanceDisk(configuration); - List endpoints = configuration.isEraseRunEndpoints() ? Collections.emptyList() : tool.getEndpoints(); + final List endpoints = configuration.isEraseRunEndpoints() + ? Collections.emptyList() : tool.getEndpoints(); configuration.setSecretName(tool.getSecretName()); final boolean sensitive = checkRunForSensitivity(configuration.getParameters()); Assert.isTrue(!sensitive || tool.isAllowSensitive(), messageHelper.getMessage( MessageConstants.ERROR_SENSITIVE_RUN_NOT_ALLOWED_FOR_TOOL, tool.getImage())); - PipelineRun run = createPipelineRun(version, configuration, pipeline, tool, toolVersion.orElse(null), region, - parentRun.orElse(null), entityIds, configurationId, sensitive); - if (parentNodeId != null && !parentNodeId.equals(run.getId())) { - setParentInstance(run, parentNodeId); + final PipelineRun run = createPipelineRun(version, configuration, pipeline, tool, toolVersion.orElse(null), + region, parentRun.orElse(null), entityIds, configurationId, sensitive); + + // If there is no podAssignPolicy we need to schedule run to be launched on dedicated node + if (configuration.getPodAssignPolicy() == null || !configuration.getPodAssignPolicy().isValid()) { + log.debug(String.format("Setup run assign policy as run id for run: %d", run.getId())); + configuration.setPodAssignPolicy( + RunAssignPolicy.builder() + .selector( + RunAssignPolicy.PodAssignSelector.builder() + .label(KubernetesConstants.RUN_ID_LABEL) + .value(run.getId().toString()).build()) + .build() + ); } - String useNodeLabel = parentNodeId != null ? parentNodeId.toString() : run.getId().toString(); + + configuration.getPodAssignPolicy() + .ifMatchThenMapValue(KubernetesConstants.RUN_ID_LABEL, Long::parseLong) + .ifPresent(parentNodeId -> { + if (!parentNodeId.equals(run.getId())) { + setParentInstance(run, parentNodeId); + } + }); + run.setConfigName(configurationName); run.setRunSids(runSids); - String launchedCommand = pipelineLauncher.launch(run, configuration, endpoints, useNodeLabel, clusterId); + final String launchedCommand = pipelineLauncher.launch(run, configuration, endpoints, clusterId); //update instance info according to evaluated command run.setActualCmd(launchedCommand); save(run); @@ -1026,8 +1051,15 @@ public PipelineRun restartRun(final PipelineRun run) { final List endpoints = configuration.isEraseRunEndpoints() ? Collections.emptyList() : tool.getEndpoints(); configuration.setSecretName(tool.getSecretName()); - final String launchedCommand = pipelineLauncher.launch(restartedRun, configuration, endpoints, - restartedRun.getId().toString(), null); + configuration.setPodAssignPolicy( + RunAssignPolicy.builder() + .selector( + RunAssignPolicy.PodAssignSelector.builder() + .label(KubernetesConstants.RUN_ID_LABEL) + .value(restartedRun.getId().toString()).build()) + .build() + ); + final String launchedCommand = pipelineLauncher.launch(restartedRun, configuration, endpoints, null); restartedRun.setActualCmd(launchedCommand); save(restartedRun); @@ -1253,12 +1285,15 @@ private void runClusterWorkers(PipelineRun run, PipelineStart runVO, String vers PipelineConfiguration configuration) { String parentId = Long.toString(run.getId()); Integer nodeCount = configuration.getNodeCount(); - configurationManager.updateWorkerConfiguration(parentId, runVO, configuration, false, true); + final PipelineConfiguration workerConfigurationTemplate = configurationManager.generateWorkerConfiguration( + parentId, runVO, configuration, false, true); for (int i = 0; i < nodeCount; i++) { - launchPipeline(configuration, pipeline, version, - runVO.getInstanceType(), runVO.getParentNodeId(), - runVO.getConfigurationName(), parentId, run.getId(), null, null, - runVO.getRunSids(), configuration.getNotifications()); + final PipelineConfiguration workerConfiguration = workerConfigurationTemplate.clone(); + launchPipeline( + workerConfiguration, pipeline, version, runVO.getInstanceType(), runVO.getConfigurationName(), + parentId, run.getId(), null, null, runVO.getRunSids(), + workerConfiguration.getNotifications() + ); } } diff --git a/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/CloudPlatformRunner.java b/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/CloudPlatformRunner.java index f33ffe2cb9..b160b4891f 100644 --- a/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/CloudPlatformRunner.java +++ b/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/CloudPlatformRunner.java @@ -164,31 +164,40 @@ private List runConfigurationEntry(RunConfigurationEntry entry, PipelineStart startVO = entry.toPipelineStart(); startVO.setNotifications(notifications); - if (!StringUtils.hasText(clusterId)) { - log.debug("Launching master entry {}", entry.getName()); - pipelineConfigurationManager.updateMasterConfiguration(configuration, startNFS); - } else { - log.debug("Launching worker entry {}", entry.getName()); - pipelineConfigurationManager.updateWorkerConfiguration(clusterId, startVO, configuration, startNFS, true); - } Pipeline pipeline = entry.getPipelineId() != null ? pipelineManager.load(entry.getPipelineId()) : null; List result = new ArrayList<>(); log.debug("Launching total {} copies of entry {}", copies, entry.getName()); + final PipelineConfiguration runConfigurationTemplate = + buildRunConfiguration(entry, configuration, clusterId, startNFS, startVO); for (int i = 0; i < copies; i++) { + final PipelineConfiguration runConfiguration = runConfigurationTemplate.clone(); //only first node may be a NFS server if (i != 0) { - configuration.setCmdTemplate(WORKER_CMD_TEMPLATE); - configuration.getParameters().remove(NFS_CLUSTER_ROLE); - configuration.buildEnvVariables(); + runConfiguration.setCmdTemplate(WORKER_CMD_TEMPLATE); + runConfiguration.getParameters().remove(NFS_CLUSTER_ROLE); + runConfiguration.buildEnvVariables(); } - result.add(pipelineRunManager.launchPipeline(configuration, pipeline, entry.getPipelineVersion(), - startVO.getInstanceType(), startVO.getParentNodeId(), - startVO.getConfigurationName(), clusterId, null, entityIds, configurationId, - startVO.getRunSids(), startVO.getNotifications())); + result.add(pipelineRunManager.launchPipeline(runConfiguration, pipeline, entry.getPipelineVersion(), + startVO.getInstanceType(), startVO.getConfigurationName(), clusterId, + null, entityIds, configurationId, startVO.getRunSids(), startVO.getNotifications())); } return result; } + private PipelineConfiguration buildRunConfiguration(final RunConfigurationEntry entry, + final PipelineConfiguration configuration, + final String clusterId, final boolean startNFS, + final PipelineStart startVO) { + if (!StringUtils.hasText(clusterId)) { + log.debug("Launching master entry {}", entry.getName()); + return pipelineConfigurationManager.generateMasterConfiguration(configuration, startNFS); + } else { + log.debug("Launching worker entry {}", entry.getName()); + return pipelineConfigurationManager + .generateWorkerConfiguration(clusterId, startVO, configuration, startNFS, true); + } + } + @Data private static class SplitConfig { diff --git a/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/FirecloudRunner.java b/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/FirecloudRunner.java index b17ff60b98..18eab46c46 100644 --- a/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/FirecloudRunner.java +++ b/api/src/main/java/com/epam/pipeline/manager/pipeline/runner/FirecloudRunner.java @@ -140,9 +140,8 @@ private PipelineRun runFirecloudAnalysis(FirecloudConfiguration settings, .build()); addCredentials(runConfiguration, settings); return pipelineRunManager.launchPipeline(runConfiguration, null, null, - startVO.getInstanceType(), startVO.getParentNodeId(), - startVO.getConfigurationName(), null, null, entities, configurationId, - startVO.getRunSids(), startVO.getNotifications()); + startVO.getInstanceType(), startVO.getConfigurationName(), null, null, + entities, configurationId, startVO.getRunSids(), startVO.getNotifications()); } private PipelineStart createFirecloudStart(FirecloudConfiguration settings, diff --git a/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java b/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java index 535ef9bc62..645543515c 100644 --- a/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java +++ b/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java @@ -128,6 +128,7 @@ public class SystemPreferences { private static final String MONITORING_GROUP = "Monitoring"; private static final String CLOUD = "Cloud"; private static final String CLOUD_REGION_GROUP = "Cloud region"; + private static final String SYSTEM_JOBS_GROUP = "System Jobs"; private static final String STORAGE_FSBROWSER_BLACK_LIST_DEFAULT = "/bin,/var,/home,/root,/sbin,/sys,/usr,/boot,/dev,/lib,/proc,/etc"; @@ -1202,6 +1203,16 @@ public class SystemPreferences { private static final Pattern GIT_VERSION_PATTERN = Pattern.compile("(\\d)\\.(\\d)"); + // System Jobs + public static final StringPreference SYSTEM_JOBS_SCRIPTS_LOCATION = new StringPreference( + "system.jobs.scripts.location", "src/system-jobs", SYSTEM_JOBS_GROUP, pass, false); + + public static final StringPreference SYSTEM_JOBS_OUTPUT_TASK = new StringPreference( + "system.jobs.output.pipeline.task", "SystemJob", SYSTEM_JOBS_GROUP, pass, false); + + public static final LongPreference SYSTEM_JOBS_PIPELINE = new LongPreference( + "system.jobs.pipeline.id", null, SYSTEM_JOBS_GROUP, isNullOrGreaterThan(0), false); + private static final Map> PREFERENCE_MAP; private ToolManager toolManager; diff --git a/api/src/main/resources/messages.properties b/api/src/main/resources/messages.properties index ec21a267af..a87bba2ab6 100644 --- a/api/src/main/resources/messages.properties +++ b/api/src/main/resources/messages.properties @@ -717,6 +717,9 @@ error.quota.action.not.allowed=Quota action ''{0}'' is not allowed for ''{1}''. error.billing.quota.exceeded.launch=Launch of new compute instances is forbidden due to exceeded billing quota. error.run.launch.user.limit.exceeded=Launch of new jobs is restricted as [{0}] user will exceed [{1}] runs limit [{2}] +error.run.assign.policy.malformed=Run assign policy is malformed: {0}. +error.run.assign.policy.forbidden=Launch of run with advanced assign policy is forbidden as [{0}] user doesn't have admin privileges. +error.run.with.service.account.forbidden=Launch of a run with specified kube service account is forbidden as [{0}] user doesn't have admin privileges. # Ngs preprocessing error.ngs.preprocessing.folder.id.is.not.provided=Folder id is not provided in the request. diff --git a/api/src/test/java/com/epam/pipeline/manager/docker/DockerContainerOperationManagerTest.java b/api/src/test/java/com/epam/pipeline/manager/docker/DockerContainerOperationManagerTest.java index f2825212ca..da42520764 100644 --- a/api/src/test/java/com/epam/pipeline/manager/docker/DockerContainerOperationManagerTest.java +++ b/api/src/test/java/com/epam/pipeline/manager/docker/DockerContainerOperationManagerTest.java @@ -213,7 +213,7 @@ public void shouldResumeRun() throws InterruptedException { verifyResumeProcessing(1); verify(cloudFacade).startInstance(REGION_ID, NODE_ID); verify(pipelineLauncher) - .launch(any(), any(), any(), anyString(), anyBoolean(), anyString(), anyString(), any()); + .launch(any(), any(), any(), anyBoolean(), anyString(), anyString(), any()); verify(runManager).updatePipelineStatus(any()); assertEquals(TaskStatus.RUNNING, run.getStatus()); } @@ -247,7 +247,7 @@ public void resumeRunShouldSkipInstanceRestart() throws InterruptedException { verifyResumeProcessing(1); verify(cloudFacade, never()).startInstance(REGION_ID, NODE_ID); verify(pipelineLauncher) - .launch(any(), any(), any(), anyString(), anyBoolean(), anyString(), anyString(), any()); + .launch(any(), any(), any(), anyBoolean(), anyString(), anyString(), any()); verify(runManager).updatePipelineStatus(any()); assertEquals(TaskStatus.RUNNING, run.getStatus()); } @@ -264,7 +264,7 @@ public void resumeRunShouldSkipConfigurationRelaunch() throws InterruptedExcepti verifyResumeProcessing(1); verify(pipelineLauncher, never()) - .launch(any(), any(), any(), anyString(), anyBoolean(), anyString(), anyString(), any()); + .launch(any(), any(), any(), anyBoolean(), anyString(), anyString(), any()); verify(runManager).updatePipelineStatus(any()); assertEquals(TaskStatus.RUNNING, run.getStatus()); } diff --git a/api/src/test/java/com/epam/pipeline/manager/execution/PipelineLauncherValidateConfigurationTest.java b/api/src/test/java/com/epam/pipeline/manager/execution/PipelineLauncherValidateConfigurationTest.java new file mode 100644 index 0000000000..080fa6536d --- /dev/null +++ b/api/src/test/java/com/epam/pipeline/manager/execution/PipelineLauncherValidateConfigurationTest.java @@ -0,0 +1,86 @@ +package com.epam.pipeline.manager.execution; + +import com.epam.pipeline.common.MessageHelper; +import com.epam.pipeline.entity.configuration.PipelineConfiguration; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; +import com.epam.pipeline.entity.user.PipelineUser; +import com.epam.pipeline.manager.cluster.KubernetesConstants; +import com.epam.pipeline.manager.security.AuthManager; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +@SuppressWarnings("PMD.UnusedPrivateField") +public class PipelineLauncherValidateConfigurationTest { + + public static final String SOME_LABEL = "some-label"; + public static final String VALUE = "true"; + public static final String KUBE_SERVICE_ACCOUNT = "some-account"; + public static final String RUN_ID_VALUE = "1"; + + @Mock + private AuthManager authManager; + @Mock + private MessageHelper messageHelper; + + @InjectMocks + private PipelineLauncher pipelineLauncher; + + @Test + public void checkRunLaunchIsNotForbiddenForAdmin() { + Mockito.doReturn(PipelineUser.builder().admin(true).build()).when(authManager).getCurrentUser(); + final PipelineConfiguration configuration = new PipelineConfiguration(); + configuration.setPodAssignPolicy( + RunAssignPolicy.builder() + .selector( + RunAssignPolicy.PodAssignSelector.builder() + .label(SOME_LABEL) + .value(VALUE).build()) + .build() + ); + configuration.setKubeServiceAccount(KUBE_SERVICE_ACCOUNT); + pipelineLauncher.validateLaunchConfiguration(configuration); + } + + @Test(expected = IllegalStateException.class) + public void checkRunLaunchWithKubeServiceAccIsForbiddenForSimpleUser() { + Mockito.doReturn(PipelineUser.builder().admin(false).build()).when(authManager).getCurrentUser(); + final PipelineConfiguration configuration = new PipelineConfiguration(); + configuration.setKubeServiceAccount(KUBE_SERVICE_ACCOUNT); + pipelineLauncher.validateLaunchConfiguration(configuration); + } + + @Test(expected = IllegalStateException.class) + public void checkRunLaunchWithAdvancedRunAssignPolicyIsForbiddenForSimpleUser() { + Mockito.doReturn(PipelineUser.builder().admin(false).build()).when(authManager).getCurrentUser(); + final PipelineConfiguration configuration = new PipelineConfiguration(); + configuration.setPodAssignPolicy( + RunAssignPolicy.builder() + .selector( + RunAssignPolicy.PodAssignSelector.builder() + .label(SOME_LABEL) + .value(VALUE).build()) + .build() + ); + pipelineLauncher.validateLaunchConfiguration(configuration); + } + + @Test + public void checkRunLaunchWithSimpleRunAssignPolicyIsAllowedForSimpleUser() { + Mockito.doReturn(PipelineUser.builder().admin(false).build()).when(authManager).getCurrentUser(); + final PipelineConfiguration configuration = new PipelineConfiguration(); + configuration.setPodAssignPolicy( + RunAssignPolicy.builder() + .selector( + RunAssignPolicy.PodAssignSelector.builder() + .label(KubernetesConstants.RUN_ID_LABEL) + .value(RUN_ID_VALUE).build()) + .build() + ); + pipelineLauncher.validateLaunchConfiguration(configuration); + } +} \ No newline at end of file diff --git a/api/src/test/java/com/epam/pipeline/manager/execution/PodSpecMapperHelperTest.java b/api/src/test/java/com/epam/pipeline/manager/execution/PodSpecMapperHelperTest.java new file mode 100644 index 0000000000..3f7f60f6a6 --- /dev/null +++ b/api/src/test/java/com/epam/pipeline/manager/execution/PodSpecMapperHelperTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.manager.execution; + +import io.fabric8.kubernetes.api.model.Toleration; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; + +public class PodSpecMapperHelperTest { + + public static final String LABEL_KEY = "label-key"; + public static final String ANOTHER_LABEL_KEY = "another-label-key"; + public static final String LABEL_VALUE = "label-value"; + public static final String EMPTY = ""; + + @Test + public void tolerationLabelWithoutValueShouldBeMappedToTolerationWithExistsOperator() { + final List tolerations = PodSpecMapperHelper + .buildTolerations(Collections.singletonMap(LABEL_KEY, EMPTY)); + Assert.assertEquals(1, tolerations.size()); + final Toleration toleration = tolerations.get(0); + Assert.assertEquals(LABEL_KEY, toleration.getKey()); + Assert.assertEquals(PodSpecMapperHelper.TOLERATION_OP_EXISTS, toleration.getOperator()); + } + + @Test + public void tolerationLabelWithoutValueShouldBeMappedToTolerationWithEqualsOperator() { + final List tolerations = PodSpecMapperHelper + .buildTolerations(Collections.singletonMap(LABEL_KEY, LABEL_VALUE)); + Assert.assertEquals(1, tolerations.size()); + final Toleration toleration = tolerations.get(0); + Assert.assertEquals(LABEL_KEY, toleration.getKey()); + Assert.assertEquals(LABEL_VALUE, toleration.getValue()); + Assert.assertEquals(PodSpecMapperHelper.TOLERATION_OP_EQUAL, toleration.getOperator()); + } + + @Test + public void shouldBePossibleToSpecifySeveralTolerations() { + final List tolerations = PodSpecMapperHelper + .buildTolerations( + new LinkedHashMap() {{ + put(LABEL_KEY, LABEL_VALUE); + put(ANOTHER_LABEL_KEY, EMPTY); + }} + ); + Assert.assertEquals(2, tolerations.size()); + Toleration toleration = tolerations.get(0); + Assert.assertEquals(LABEL_KEY, toleration.getKey()); + Assert.assertEquals(LABEL_VALUE, toleration.getValue()); + Assert.assertEquals(PodSpecMapperHelper.TOLERATION_OP_EQUAL, toleration.getOperator()); + + toleration = tolerations.get(1); + Assert.assertEquals(ANOTHER_LABEL_KEY, toleration.getKey()); + Assert.assertEquals(PodSpecMapperHelper.TOLERATION_OP_EXISTS, toleration.getOperator()); + } +} diff --git a/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManagerTest.java b/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManagerTest.java index 00416505f2..987c4fbfb5 100644 --- a/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManagerTest.java +++ b/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineConfigurationManagerTest.java @@ -20,8 +20,11 @@ import com.epam.pipeline.entity.configuration.PipelineConfiguration; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; +import com.epam.pipeline.manager.cluster.KubernetesConstants; import com.epam.pipeline.manager.preference.PreferenceManager; import com.epam.pipeline.manager.region.CloudRegionManager; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.InjectMocks; @@ -45,6 +48,7 @@ import static org.mockito.Mockito.verify; public class PipelineConfigurationManagerTest { + public static final String NODE_LABEL_VALUE = "true"; private static final String TEST_IMAGE = "image"; private static final String TEST_REPO = "repository"; private static final String OWNER1 = "testUser1"; @@ -63,6 +67,8 @@ public class PipelineConfigurationManagerTest { private static final String TEST_PATH_2 = "test/path2"; private static final String TEST_PATH_3 = "test/path3"; private static final String TEST_PATH_4 = "test/path4"; + public static final String NODE_LABEL = "node-label"; + public static final String OTHER_NODE_LABEL_VALUE = "false"; @Mock private PipelineVersionManager pipelineVersionManager; @@ -112,4 +118,87 @@ public void shouldGetUnregisteredPipelineConfiguration() { verify(pipelineVersionManager).getValidDockerImage(eq(TEST_IMAGE)); } + + @Test + public void shouldPropagateRunAssignPolicyFromStartObjectToConfiguration() { + final PipelineStart runVO = getPipelineStart(TEST_PARAMS, TEST_IMAGE); + runVO.setPodAssignPolicy( + RunAssignPolicy.builder().selector( + RunAssignPolicy.PodAssignSelector.builder().label(NODE_LABEL).value(NODE_LABEL_VALUE).build() + ).build() + ); + final PipelineConfiguration config = pipelineConfigurationManager + .mergeParameters(runVO, new PipelineConfiguration()); + + Assert.assertNotNull(config.getPodAssignPolicy()); + Assert.assertEquals(NODE_LABEL, config.getPodAssignPolicy().getSelector().getLabel()); + Assert.assertEquals(NODE_LABEL_VALUE, config.getPodAssignPolicy().getSelector().getValue()); + } + + @Test + public void shouldPropagateParentNodeIdOrUseRunIdFromStartObjectToRunAssignPolicyInConfiguration() { + PipelineStart runVO = getPipelineStart(TEST_PARAMS, TEST_IMAGE); + runVO.setParentNodeId(2L); + PipelineConfiguration config = pipelineConfigurationManager.mergeParameters(runVO, new PipelineConfiguration()); + + Assert.assertNotNull(config.getPodAssignPolicy()); + Assert.assertEquals(KubernetesConstants.RUN_ID_LABEL, config.getPodAssignPolicy().getSelector().getLabel()); + Assert.assertEquals("2", config.getPodAssignPolicy().getSelector().getValue()); + + runVO = getPipelineStart(TEST_PARAMS, TEST_IMAGE); + runVO.setUseRunId(3L); + config = pipelineConfigurationManager.mergeParameters(runVO, new PipelineConfiguration()); + + Assert.assertNotNull(config.getPodAssignPolicy()); + Assert.assertEquals(KubernetesConstants.RUN_ID_LABEL, config.getPodAssignPolicy().getSelector().getLabel()); + Assert.assertEquals("3", config.getPodAssignPolicy().getSelector().getValue()); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldFailIfBothRunAssignPolicyOrParentNodeIdIsProvided() { + final PipelineStart runVO = getPipelineStart(TEST_PARAMS, TEST_IMAGE); + runVO.setPodAssignPolicy( + RunAssignPolicy.builder().selector( + RunAssignPolicy.PodAssignSelector.builder().label(NODE_LABEL).value(NODE_LABEL_VALUE).build() + ).build() + ); + runVO.setParentNodeId(1L); + pipelineConfigurationManager.mergeParameters(runVO, new PipelineConfiguration()); + } + + @Test + public void shouldPreferToUseValuesFromStartObjectInMergedConfiguration() { + PipelineStart runVO = getPipelineStart(TEST_PARAMS, TEST_IMAGE); + runVO.setParentNodeId(2L); + PipelineConfiguration defaultConfig = new PipelineConfiguration(); + defaultConfig.setPodAssignPolicy( + RunAssignPolicy.builder().selector( + RunAssignPolicy.PodAssignSelector.builder().label(NODE_LABEL).value(NODE_LABEL_VALUE).build() + ).build() + ); + PipelineConfiguration config = pipelineConfigurationManager.mergeParameters(runVO, defaultConfig); + + Assert.assertNotNull(config.getPodAssignPolicy()); + Assert.assertEquals(KubernetesConstants.RUN_ID_LABEL, config.getPodAssignPolicy().getSelector().getLabel()); + Assert.assertEquals("2", config.getPodAssignPolicy().getSelector().getValue()); + + runVO = getPipelineStart(TEST_PARAMS, TEST_IMAGE); + runVO.setPodAssignPolicy( + RunAssignPolicy.builder().selector( + RunAssignPolicy.PodAssignSelector.builder().label(NODE_LABEL).value(NODE_LABEL_VALUE).build() + ).build() + ); + defaultConfig = new PipelineConfiguration(); + defaultConfig.setPodAssignPolicy( + RunAssignPolicy.builder().selector( + RunAssignPolicy.PodAssignSelector.builder().label(NODE_LABEL) + .value(OTHER_NODE_LABEL_VALUE).build() + ).build() + ); + config = pipelineConfigurationManager.mergeParameters(runVO, defaultConfig); + + Assert.assertNotNull(config.getPodAssignPolicy()); + Assert.assertEquals(NODE_LABEL, config.getPodAssignPolicy().getSelector().getLabel()); + Assert.assertEquals(NODE_LABEL_VALUE, config.getPodAssignPolicy().getSelector().getValue()); + } } diff --git a/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerLaunchTest.java b/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerLaunchTest.java index 37846c7603..c55d25e591 100644 --- a/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerLaunchTest.java +++ b/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerLaunchTest.java @@ -456,7 +456,7 @@ private PipelineRun launchPipelineWithNotificationRequests(final PipelineConfigu private PipelineRun launchPipeline(final PipelineConfiguration configuration, final Pipeline pipeline, final String instanceType, final Long parentRunId, final List notificationRequests) { - return pipelineRunManager.launchPipeline(configuration, pipeline, null, instanceType, null, + return pipelineRunManager.launchPipeline(configuration, pipeline, null, instanceType, null, null, parentRunId, null, null, null, notificationRequests); } diff --git a/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerTest.java b/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerTest.java index e6d06a8050..886daef94b 100644 --- a/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerTest.java +++ b/api/src/test/java/com/epam/pipeline/manager/pipeline/PipelineRunManagerTest.java @@ -164,8 +164,7 @@ public void setUp() throws Exception { when(instanceOfferManager.isPriceTypeAllowed(anyString(), any(), anyBoolean())).thenReturn(true); when(instanceOfferManager.getInstanceEstimatedPrice(anyString(), anyInt(), anyBoolean(), anyLong())) .thenReturn(price); - when(pipelineLauncher.launch(any(PipelineRun.class), any(), any(), anyString(), anyString())) - .thenReturn("sleep"); + when(pipelineLauncher.launch(any(PipelineRun.class), any(), any(), anyString())).thenReturn("sleep"); when(toolScanInfoManager.loadToolVersionScanInfo(notScannedTool.getId(), null)) .thenReturn(Optional.empty()); final ToolVersion toolVersion = ToolVersion.builder().size(1L).platform("linux").build(); diff --git a/api/src/test/java/com/epam/pipeline/test/aspect/AspectTestBeans.java b/api/src/test/java/com/epam/pipeline/test/aspect/AspectTestBeans.java index d38c1d1c3f..3e58d9d1bc 100644 --- a/api/src/test/java/com/epam/pipeline/test/aspect/AspectTestBeans.java +++ b/api/src/test/java/com/epam/pipeline/test/aspect/AspectTestBeans.java @@ -413,6 +413,9 @@ public class AspectTestBeans { @MockBean protected CacheManager cacheManager; + @MockBean(name = "aclCacheManager") + protected CacheManager aclCacheManager; + @MockBean protected NatGatewayDao natGatewayDao; diff --git a/client/src/components/main/AppRouter.js b/client/src/components/main/AppRouter.js index a7ebc100ca..a22ca41d4c 100644 --- a/client/src/components/main/AppRouter.js +++ b/client/src/components/main/AppRouter.js @@ -77,6 +77,7 @@ import ProjectHistory from '../pipelines/browser/ProjectHistory'; import {FacetedSearchPage} from '../search'; import {HcsImagePage} from '../special/hcs-image'; import NotificationBrowser from './notification/NotificationBrowser'; +import TicketsBrowser from '../special/tickets'; function HomePageRedirectionComponent ({router, uiNavigation}) { if (uiNavigation.loaded && router) { @@ -110,6 +111,7 @@ function AppRouterComponent ({history, uiNavigation}) { path="/folder/:folder/metadata/:entity/redirect" component={MetadataClassEntityRedirection} /> + diff --git a/client/src/components/main/home/panels/components/renderRunCard.js b/client/src/components/main/home/panels/components/renderRunCard.js index 6decf25a50..aca62ab617 100644 --- a/client/src/components/main/home/panels/components/renderRunCard.js +++ b/client/src/components/main/home/panels/components/renderRunCard.js @@ -63,7 +63,11 @@ function renderPipeline (run) { displayName = parts[parts.length - 1]; } let clusterIcon; - if (run.nodeCount > 0) { + if ( + run.nodeCount > 0 || + run.clusterRun || + run.childRunsCount > 0 + ) { clusterIcon = ; } const runName = ( diff --git a/client/src/components/main/navigation/Navigation.js b/client/src/components/main/navigation/Navigation.js index 6961d6534b..278838c3a3 100644 --- a/client/src/components/main/navigation/Navigation.js +++ b/client/src/components/main/navigation/Navigation.js @@ -330,6 +330,7 @@ export default class Navigation extends React.Component { {menuItems} { @@ -95,6 +102,51 @@ class SupportMenuItem extends React.Component { }); }; + openCreateTicketModal = () => { + this.setState({ + createTicketModalVisible: true + }); + }; + + closeCreateTicketModal = () => { + this.setState({ + createTicketModalVisible: false + }); + }; + + createTicket = (payload) => { + this.setState({ticketPending: true}, async () => { + const hide = message.loading(`Creating ticket...`, 0); + try { + const request = new GitlabIssueCreate(); + await request.send(payload); + if (request.error) { + throw new Error(request.error); + } + this.setState({ + ticketPending: false, + createTicketModalVisible: false + }, () => this.navigateToTickets()); + } catch (error) { + message.error(error.message, 5); + this.setState({ + ticketPending: false + }); + } finally { + hide(); + } + }); + }; + + navigateToTickets = () => { + const { + router + } = this.props; + if (router && typeof router.push === 'function') { + router.push('/tickets'); + } + }; + renderIcon = () => { const {icon} = this.props; if (urlCheck(icon)) { @@ -128,6 +180,12 @@ class SupportMenuItem extends React.Component { case actions.hcs: this.openHCSJobs(); break; + case actions.createTicket: + this.openCreateTicketModal(); + break; + case actions.openTickets: + this.navigateToTickets(); + break; } }; @@ -179,6 +237,14 @@ class SupportMenuItem extends React.Component { visible={this.state.hcsJobsModalVisible} onClose={this.closeHCSJobs} /> + ); diff --git a/client/src/components/main/navigation/support-menu/index.js b/client/src/components/main/navigation/support-menu/index.js index 06f2896858..1e2f147d5c 100644 --- a/client/src/components/main/navigation/support-menu/index.js +++ b/client/src/components/main/navigation/support-menu/index.js @@ -34,7 +34,8 @@ class SupportMenu extends React.Component { containerClassName: PropTypes.string, itemClassName: PropTypes.string, containerStyle: PropTypes.object, - itemStyle: PropTypes.object + itemStyle: PropTypes.object, + router: PropTypes.object }; state = { @@ -130,6 +131,7 @@ class SupportMenu extends React.Component { target={target} url={url} hint={hint} + router={this.props.router} /> ); diff --git a/client/src/components/runs/AllRuns.js b/client/src/components/runs/AllRuns.js index f768962a2d..2d96df6e4e 100644 --- a/client/src/components/runs/AllRuns.js +++ b/client/src/components/runs/AllRuns.js @@ -126,7 +126,8 @@ class AllRuns extends React.Component { const counter = continuousFetch({ fetchImmediate: true, call, - afterInvoke: after + afterInvoke: after, + intervalMS: 10000 }); this.counters[filter.key] = { counter, @@ -246,7 +247,10 @@ class AllRuns extends React.Component { ); } - const filters = {...(current.filters || {})}; + const filters = { + ...(current.filters || {}), + onlyMasterJobs: true + }; if ( current.showPersonalRuns && !all && diff --git a/client/src/components/runs/logs/Log.js b/client/src/components/runs/logs/Log.js index 7afb441a61..7bce253f84 100644 --- a/client/src/components/runs/logs/Log.js +++ b/client/src/components/runs/logs/Log.js @@ -143,7 +143,9 @@ class Logs extends localization.LocalizedReactComponent { error: undefined, showActiveWorkersOnly: false, nestedRuns: [], + hasNestedRuns: false, totalNestedRuns: 0, + nestedRunsPending: false, runTasks: [], language: undefined, timings: false, @@ -192,7 +194,9 @@ class Logs extends localization.LocalizedReactComponent { pending: true, error: undefined, nestedRuns: [], + hasNestedRuns: false, totalNestedRuns: 0, + nestedRunsPending: false, showActiveWorkersOnly: false, runTasks: [], language: undefined @@ -226,7 +230,9 @@ class Logs extends localization.LocalizedReactComponent { pending: false, error: undefined, nestedRuns: [], + hasNestedRuns: false, totalNestedRuns: 0, + nestedRunsPending: false, showActiveWorkersOnly: false, runTasks: [], language: undefined @@ -1339,10 +1345,12 @@ class Logs extends localization.LocalizedReactComponent { renderNestedRuns = () => { const { nestedRuns: originalNestedRuns = [], - totalNestedRuns: total = 0, + hasNestedRuns, + totalNestedRuns = 0, + nestedRunsPending, showActiveWorkersOnly } = this.state; - if (originalNestedRuns.length === 0) { + if (!hasNestedRuns) { return null; } const nestedRuns = originalNestedRuns.slice().sort((rA, rB) => { @@ -1390,10 +1398,15 @@ class Logs extends localization.LocalizedReactComponent { ); }; const searchParts = [`parent.id=${this.props.runId}`]; + const search = searchParts.join(' and '); + const nestedRunsInfos = [ + totalNestedRuns, + 'nested' + ]; if (showActiveWorkersOnly) { - searchParts.push('status=RUNNING'); + nestedRunsInfos.push('active'); } - const search = searchParts.join(' and '); + nestedRunsInfos.push(totalNestedRuns === 1 ? 'run' : 'runs'); return ( Nested runs: - - {nestedRuns.map(renderSingleRun)} + { - total > MAX_NESTED_RUNS_TO_DISPLAY && + !nestedRunsPending && ( +
+ {nestedRunsInfos.join(' ')} +
+ ) + } +
+ { + nestedRuns.length === 0 && nestedRunsPending && ( + + ) + } + {nestedRuns.map(renderSingleRun)} - show all {total} runs + show all nested runs - } +
); diff --git a/client/src/components/runs/logs/misc/fetch-run-info.js b/client/src/components/runs/logs/misc/fetch-run-info.js index ed1c2040ce..1eacb90cef 100644 --- a/client/src/components/runs/logs/misc/fetch-run-info.js +++ b/client/src/components/runs/logs/misc/fetch-run-info.js @@ -18,6 +18,7 @@ import PipelineRunInfo from '../../../../models/pipelines/PipelineRunInfo'; import PipelineRunSingleFilter from '../../../../models/pipelines/PipelineRunSingleFilter'; import RunTasks from '../../../../models/pipelines/RunTasks'; import PipelineLanguage from '../../../../models/pipelines/PipelineLanguage'; +import RunCount, {ALL_STATUSES} from '../../../../models/pipelines/RunCount'; import continuousFetch from '../../../../utils/continuous-fetch'; import {checkCommitAllowedForTool} from '../../actions'; @@ -56,11 +57,18 @@ export default async function fetchRunInfo ( let stopped = false; const runInfo = new PipelineRunInfo(runIdentifier); const runTasks = new RunTasks(runIdentifier); + const totalNestedRuns = new RunCount({ + parentId: Number(runIdentifier), + statuses: ALL_STATUSES, + onlyMasterJobs: false + }); await Promise.all([ runInfo.fetch(), runTasks.fetch(), + totalNestedRuns.fetch(), dockerRegistries ? dockerRegistries.fetchIfNeededOrWait() : false ].filter(Boolean)); + let hasNestedRuns = totalNestedRuns.runsCount > 0; if (runInfo.error || !runInfo.loaded) { throw new Error(runInfo.error || 'Error fetching run info'); } @@ -78,7 +86,10 @@ export default async function fetchRunInfo ( runTasks: runTasks.value || [], showActiveWorkersOnly, language: undefined, - commitAllowed + commitAllowed, + hasNestedRuns, + totalNestedRuns: 0, + nestedRunsPending: true }); } const nestedRuns = new PipelineRunSingleFilter( @@ -120,11 +131,14 @@ export default async function fetchRunInfo ( } = run; currentStatus = nextStatus; const error = runInfo.error; + hasNestedRuns = hasNestedRuns || (nestedRuns.total || 0) > 0; if (typeof dataCallback === 'function' && !stopped) { dataCallback({ run, nestedRuns: nestedRuns.value || [], + hasNestedRuns, totalNestedRuns: nestedRuns.total || 0, + nestedRunsPending: false, error, showActiveWorkersOnly, runTasks: runTasks.value || [], diff --git a/client/src/components/runs/run-table/index.js b/client/src/components/runs/run-table/index.js index 55e4a063e4..4220445424 100644 --- a/client/src/components/runs/run-table/index.js +++ b/client/src/components/runs/run-table/index.js @@ -24,7 +24,7 @@ import { filtersAreEqual, getFiltersPayload, simpleArraysAreEqual -} from './filter'; +} from '../../../models/pipelines/pipeline-runs-filter'; import { AllColumns, Columns, diff --git a/client/src/components/special/tickets/index.js b/client/src/components/special/tickets/index.js new file mode 100644 index 0000000000..d1c923bdc4 --- /dev/null +++ b/client/src/components/special/tickets/index.js @@ -0,0 +1,235 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import classNames from 'classnames'; +import {inject, observer} from 'mobx-react'; +import {computed} from 'mobx'; +import { + Button, + message, + Icon +} from 'antd'; +import GitlabIssueDelete from '../../../models/gitlab-issues/GitlabIssueDelete'; +import GitlabIssueCreate from '../../../models/gitlab-issues/GitlabIssueCreate'; +import {TicketsList, Ticket, NewTicketForm} from './special'; +import styles from './tickets-browser.css'; + +const MODES = { + list: 'list', + editTicket: 'editTicket', + createNewTicket: 'createNewTicket' +}; + +@inject((stores, {params}) => { + return { + ticketId: params.id + }; +}) +@observer +export default class TicketsBrowser extends React.Component { + state = { + pending: false, + showNewTicketModal: false, + refreshTokenId: 0 + } + + get mode () { + const {ticketId} = this.props; + if (ticketId === 'new') { + return MODES.createNewTicket; + } + if (ticketId !== undefined && ticketId !== 'new') { + return MODES.editTicket; + } + return MODES.list; + } + + @computed + get ticketId () { + const {ticketId} = this.props; + return ticketId; + } + + createTicket = (payload, shouldNavigateBack = false) => { + this.setState({pending: true}, async () => { + const request = new GitlabIssueCreate(); + const hide = message.loading(`Creating ticket...`, 0); + await request.send(payload); + this.closeNewTicketModal(); + hide(); + this.setState({ + pending: false, + refreshTokenId: this.state.refreshTokenId + 1 + }, () => { + if (request.error) { + message.error(request.error, 5); + } else { + shouldNavigateBack && this.navigateBack(); + } + }); + }); + }; + + deleteTicket = (id, shouldNavigateBack = false) => { + if (!id) { + return null; + } + this.setState({pending: true}, async () => { + const request = new GitlabIssueDelete(id); + const hide = message.loading(`Deleting ticket ${id}...`, 0); + await request.fetch(); + hide(); + this.setState({ + pending: false, + refreshTokenId: this.state.refreshTokenId + 1 + }, () => { + if (request.error) { + message.error(request.error, 5); + } else { + shouldNavigateBack && this.navigateBack(); + } + }); + }); + }; + + showNewTicketModal = () => { + this.setState({showNewTicketModal: true}); + }; + + closeNewTicketModal = () => { + this.setState({showNewTicketModal: false}); + }; + + onSelectTicket = (iid) => { + const {router} = this.props; + router && router.push(`/tickets/${iid}`); + }; + + navigateBack = () => { + const {router} = this.props; + router && router.push(`/tickets`); + }; + + renderHeader = () => { + const goBackButton = ( + + ); + const content = { + [MODES.createNewTicket]: [ + goBackButton, + + Create new ticket + + ], + [MODES.editTicket]: [ + goBackButton, + + Edit ticket + + ], + [MODES.list]: [ + + Tickets list + , + + ] + }; + return ( +
+ {content[this.mode] || null} +
); + }; + + renderContent = () => { + const {pending, refreshTokenId} = this.state; + const content = { + [MODES.createNewTicket]: ( + + ), + [MODES.editTicket]: ( + + ), + [MODES.list]: ( + + ) + }; + return content[this.mode] || null; + }; + + render () { + const {showNewTicketModal, pending} = this.state; + return ( +
+ {this.renderHeader()} + {this.renderContent()} + {showNewTicketModal ? ( + + ) : null} +
+ ); + } +} diff --git a/client/src/components/special/tickets/special/comment-card/index.js b/client/src/components/special/tickets/special/comment-card/index.js new file mode 100644 index 0000000000..b73077d2ec --- /dev/null +++ b/client/src/components/special/tickets/special/comment-card/index.js @@ -0,0 +1,175 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import classNames from 'classnames'; +import { + Icon, + Dropdown, + Menu +} from 'antd'; +import Markdown from '../../../markdown'; +import displayDate from '../../../../../utils/displayDate'; +import getAuthor from '../utilities/get-author'; +import UserName from '../../../UserName'; + +export default class CommentCard extends React.Component { + static propTypes = { + comment: PropTypes.object, + className: PropTypes.string, + onSelectMenu: PropTypes.func, + headerClassName: PropTypes.string, + style: PropTypes.object + }; + + getCommentInfo () { + const {comment} = this.props; + if (!comment) { + return { + author: undefined, + text: '' + }; + } + const { + author: systemAuthor = {}, + description, + body, + type + } = comment; + let { + author = '' + } = systemAuthor.name; + let text = description || body; + if (/^issue$/i.test(type)) { + author = getAuthor(comment); + } else { + const authorLabel = text + .split('\n') + .find(part => part.toLowerCase().includes('on behalf of')); + if (authorLabel) { + author = authorLabel.split('of').pop().trim(); + text = text.replace(authorLabel, ''); + } + } + return { + author, + text + }; + } + + onSelectMenu = (key, comment) => { + const {onSelectMenu} = this.props; + onSelectMenu && onSelectMenu(key, comment); + }; + + render () { + const { + comment, + className, + headerClassName, + onSelectMenu, + style + } = this.props; + if (!comment) { + return null; + } + const { + author, + text + } = this.getCommentInfo(); + const messageMenu = ( + this.onSelectMenu(key, comment)} + selectedKeys={[]} + style={{cursor: 'pointer'}} + > + + Edit + + + Delete + + + ); + return ( +
+
+
+ + + commented {displayDate(comment.updated_at, 'D MMM YYYY, HH:mm')} + +
+ {onSelectMenu ? ( + + + + ) : null} +
+ +
+ ); + } +} diff --git a/client/src/components/special/tickets/special/comment-editor/index.js b/client/src/components/special/tickets/special/comment-editor/index.js new file mode 100644 index 0000000000..8ee4e4ea9b --- /dev/null +++ b/client/src/components/special/tickets/special/comment-editor/index.js @@ -0,0 +1,227 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import classNames from 'classnames'; +import { + Input, + Radio, + Upload, + Button, + Icon +} from 'antd'; +import Markdown from '../../../markdown'; +import blobFilesToBase64 from '../utilities/blob-files-to-base64'; + +const MODES = { + edit: 'edit', + preview: 'preview' +}; + +export default class CommentEditor extends React.Component { + static propTypes = { + isNewComment: PropTypes.bool, + comment: PropTypes.object, + onCancel: PropTypes.func, + onSave: PropTypes.func, + className: PropTypes.string, + disabled: PropTypes.bool + }; + + state={ + mode: MODES.edit, + description: '', + fileList: [] + } + + componentDidMount () { + this.setInitialState(); + } + + componentDidUpdate (prevProps) { + if (prevProps.comment?.id !== this.props.comment?.id) { + this.setInitialState(); + } + } + + get submitDisabled () { + const {disabled} = this.props; + const {description} = this.state; + return disabled || !description; + } + + setInitialState = () => { + const {comment} = this.props; + if (comment) { + this.setState({ + description: comment.description || comment.body + }); + } + }; + + clearState = () => { + this.setState({ + description: '', + fileList: [] + }); + } + + onModeChange = (event) => { + this.setState({mode: event.target.value}); + }; + + onChangeTextField = fieldName => event => { + this.setState({[fieldName]: event.target.value}); + }; + + onCancel = () => { + const {onCancel} = this.props; + onCancel && onCancel(); + }; + + onSave = async () => { + const {description, fileList} = this.state; + const {onSave, isNewComment} = this.props; + const base64Files = fileList && fileList.length > 0 + ? await blobFilesToBase64(fileList) + : {}; + onSave && onSave({ + description, + ...(Object.keys(base64Files).length > 0 && {attachments: base64Files}) + }); + isNewComment && this.clearState(); + }; + + onRemoveFile = (file) => { + this.setState(({fileList}) => { + const index = fileList.indexOf(file); + const newFileList = fileList.slice(); + newFileList.splice(index, 1); + return { + fileList: newFileList + }; + }); + }; + + beforeUpload = (file) => { + this.setState(({fileList}) => ({ + fileList: [...fileList, file] + })); + return false; + }; + + render () { + const {mode, description, fileList} = this.state; + const { + isNewComment, + className, + uploadEnabled, + disabled + } = this.props; + return ( +
+ + {Object.values(MODES).map((key) => ( + + {key} + + ))} + +
+ {mode === MODES.edit ? ( +
+
+ e.stopPropagation()} + placeholder="Leave a comment" + style={{ + resize: 'none', + borderRadius: '0 4px 0 0' + }} + /> +
+ {!isNewComment ? () : null} + +
+
+ {uploadEnabled ? ( + + + + ) : ()} +
+ ) : ( + + )} +
+
+ ); + } +} diff --git a/client/src/components/special/tickets/special/index.js b/client/src/components/special/tickets/special/index.js new file mode 100644 index 0000000000..5abf20fce9 --- /dev/null +++ b/client/src/components/special/tickets/special/index.js @@ -0,0 +1,25 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import TicketsList from './tickets-list'; +import NewTicketForm from './new-ticket-form'; +import Ticket from './ticket'; + +export { + TicketsList, + NewTicketForm, + Ticket +}; diff --git a/client/src/components/special/tickets/special/label/index.js b/client/src/components/special/tickets/special/label/index.js new file mode 100644 index 0000000000..9c1c08d3d1 --- /dev/null +++ b/client/src/components/special/tickets/special/label/index.js @@ -0,0 +1,48 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import classNames from 'classnames'; +import styles from './label.css'; + +export const STATUS_COLORS = { + opened: 'cp-primary border', + open: 'cp-primary border', + closed: 'cp-success border', + close: 'cp-success border', + inprogress: 'cp-warning border', + 'in-progress': 'cp-warning border', + 'in progress': 'cp-warning border' +}; + +export default function Label ({label = '', style = {}, className}) { + if (!label) { + return null; + } + return ( +
+ {label} +
+ ); +}; diff --git a/client/src/components/special/tickets/special/label/label.css b/client/src/components/special/tickets/special/label/label.css new file mode 100644 index 0000000000..9eca5f2ae2 --- /dev/null +++ b/client/src/components/special/tickets/special/label/label.css @@ -0,0 +1,28 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.label { + border-radius: 4px; + opacity: 0.9; + display: flex; + align-items: center; + justify-content: center; + padding: 0 5px; +} + +.label:not(:last-child) { + margin-right: 3px; +} diff --git a/client/src/components/special/tickets/special/new-ticket-form/index.js b/client/src/components/special/tickets/special/new-ticket-form/index.js new file mode 100644 index 0000000000..9ef6a9bafd --- /dev/null +++ b/client/src/components/special/tickets/special/new-ticket-form/index.js @@ -0,0 +1,288 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import { + Input, + Radio, + Upload, + Button, + Icon, + Modal +} from 'antd'; +import Markdown from '../../../markdown'; +import blobFilesToBase64 from '../utilities/blob-files-to-base64'; +import styles from './new-ticket-form.css'; + +const PREVIEW_MODES = { + edit: 'edit', + preview: 'preview' +}; + +export default class NewTicketForm extends React.Component { + static propTypes = { + onSave: PropTypes.func, + onCancel: PropTypes.func, + title: PropTypes.string, + pending: PropTypes.bool, + renderAsModal: PropTypes.bool, + modalVisible: PropTypes.bool + }; + + state = { + description: '', + title: '', + mode: PREVIEW_MODES.edit, + fileList: [] + } + + componentDidUpdate (prevProps, prevState, snapshot) { + if ( + prevProps.modalVisible !== this.props.modalVisible && + this.props.renderAsModal + ) { + this.reset(); + } + } + + reset = () => { + this.setState({ + description: '', + title: '', + mode: PREVIEW_MODES.edit, + fileList: [] + }); + }; + + get submitDisabled () { + const {description, title} = this.state; + const {pending} = this.props; + return pending || !title || !description; + } + + onChangeMode = (event) => { + this.setState({mode: event.target.value}); + }; + + onChangeValue = (field, eventType, stopPropagation = false) => event => { + event && stopPropagation && event.stopPropagation(); + let value; + switch (eventType) { + case 'input': + value = event.target.value; + break; + case 'select': + value = event; + break; + default: + value = undefined; + } + if (value !== undefined) { + this.setState({[field]: value}); + } + }; + + onRemoveFile = (file) => { + this.setState(({fileList}) => { + const index = fileList.indexOf(file); + const newFileList = fileList.slice(); + newFileList.splice(index, 1); + return { + fileList: newFileList + }; + }); + }; + + beforeUpload = (file) => { + this.setState(({fileList}) => ({ + fileList: [...fileList, file] + })); + return false; + }; + + onCancelClick = () => { + const {onCancel} = this.props; + onCancel && onCancel(); + }; + + onSubmitClick = async () => { + const { + title, + description, + fileList, + renderAsModal + } = this.state; + const {onSave} = this.props; + const base64Files = fileList && fileList.length > 0 + ? await blobFilesToBase64(fileList) + : {}; + const payload = { + title, + description, + ...(Object.keys(base64Files).length > 0 && {attachments: base64Files}) + }; + onSave && onSave(payload, !renderAsModal); + }; + + renderPreview = () => { + const {description} = this.state; + if (!description) { + return ( + + Nothing to preview. + + ); + } + return ( + + ); + }; + + renderTicketEditor = () => { + const { + mode, + description, + title, + fileList + } = this.state; + const {uploadEnabled, renderAsModal} = this.props; + return ( +
+ +
+ + {Object.values(PREVIEW_MODES).map((key) => ( + + {key} + + ))} + +
+
+ {mode === PREVIEW_MODES.edit ? ( +
+
+ e.stopPropagation()} + placeholder="Leave a comment" + style={{ + resize: 'none', + borderRadius: '0 4px 0 0' + }} + /> + {!renderAsModal ? ( + + ) : null} +
+ {uploadEnabled ? ( + + + + ) :
} +
+ ) : ( + this.renderPreview() + )} +
+
+ ); + }; + + render () { + const {renderAsModal, modalVisible, title} = this.props; + if (renderAsModal) { + return ( + + + +
+ )} + > +
+
+ {this.renderTicketEditor()} +
+
+ + ); + } + return ( +
+
+ {this.renderTicketEditor()} +
+
+ ); + } +} diff --git a/client/src/components/special/tickets/special/new-ticket-form/new-ticket-form.css b/client/src/components/special/tickets/special/new-ticket-form/new-ticket-form.css new file mode 100644 index 0000000000..e28a5ca9a9 --- /dev/null +++ b/client/src/components/special/tickets/special/new-ticket-form/new-ticket-form.css @@ -0,0 +1,61 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.container { + display: flex; + justify-content: center; +} + +.content { + display: flex; + flex-wrap: nowrap; + max-width: 900px; + width: 100%; +} + +.preview-placeholder { + line-height: 32px; + padding: 5px; +} + +.editor-container { + flex: 1 1 0; + max-width: 100%; +} + +.editor-form { + display: flex; + flex-direction: column; +} + +.submit-button { + position: absolute; + bottom: -28px; + right: 0; + height: 28px; + border-radius: 0 0 4px 4px; +} + +.upload-button { + border-radius: 0 0 4px 4px; +} + +.modal-footer { + display: flex; + flex-direction: row; + align-items: center; + justify-content: flex-end; +} diff --git a/client/src/components/special/tickets/special/ticket/index.js b/client/src/components/special/tickets/special/ticket/index.js new file mode 100644 index 0000000000..3786781302 --- /dev/null +++ b/client/src/components/special/tickets/special/ticket/index.js @@ -0,0 +1,274 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import classNames from 'classnames'; +import {computed, observable} from 'mobx'; +import {observer, inject} from 'mobx-react'; +import {Alert, message, Spin} from 'antd'; +import moment from 'moment-timezone'; +import CommentCard from '../comment-card'; +import CommentEditor from '../comment-editor'; +import Label from '../label'; +import getAuthor from '../utilities/get-author'; +import GitlabIssueLoad from '../../../../../models/gitlab-issues/GitlabIssueLoad'; +import GitlabIssueComment from '../../../../../models/gitlab-issues/GitlabIssueComment'; +import UserName from '../../../UserName'; +import styles from './ticket.css'; + +@inject('preferences') +@observer +export default class Ticket extends React.Component { + static propTypes = { + ticketId: PropTypes.oneOfType(PropTypes.string, PropTypes.number), + pending: PropTypes.bool, + onNavigateBack: PropTypes.func, + onSaveComment: PropTypes.func + }; + + state = { + editComment: undefined, + pending: false + } + + @observable + ticketRequest; + + editorRef; + + componentDidMount () { + const {ticketId} = this.props; + this.fetchTicket(ticketId); + }; + + componentDidUpdate (prevProps) { + if (this.props.ticketId !== prevProps.ticketId) { + this.fetchTicket(this.props.ticketId); + } + } + + @computed + get predefinedLabels () { + const {preferences} = this.props; + if (preferences && preferences.loaded) { + return (preferences.gitlabIssueStatuses || []) + .map(status => status.toLowerCase()); + } + return []; + } + + @computed + get ticket () { + if (this.ticketRequest && this.ticketRequest.loaded) { + return (this.ticketRequest || {}).value; + } + return null; + } + + @computed + get comments () { + if (this.ticket) { + return [...(this.ticket.comments || [])] + .sort((a, b) => moment.utc(a.created_at) - moment.utc(b.created_at)); + } + return []; + } + + get pending () { + return this.props.pending || this.state.pending; + } + + onSelectCommentMenu = (key, comment) => { + if (key === 'edit') { + return this.setState({editComment: comment.iid || comment.id}); + } + }; + + onCancelEditing = () => { + this.setState({editComment: undefined}); + }; + + onSaveCommentEditing = () => { + // API does not support comment editing at the moment + }; + + onSaveNewComment = ({description}) => { + this.setState({pending: true}, async () => { + const {iid} = this.ticket; + const request = new GitlabIssueComment(iid); + const hide = message.loading(`Creating comment...`, 0); + await request.send({body: description}); + await this.ticketRequest.fetch(); + hide(); + this.setState({pending: false}, () => { + if (request.error) { + message.error(request.error, 5); + this.ticketRequest = undefined; + } else { + this.scrollToEditor(); + } + }); + }); + }; + + fetchTicket = (ticketId) => { + if (!ticketId) { + return; + } + this.setState({pending: true}, async () => { + this.ticketRequest = new GitlabIssueLoad(ticketId); + const hide = message.loading(`Fetching ticket ${ticketId}...`, 0); + await this.ticketRequest.fetch(); + if (this.ticketRequest.error) { + message.error(this.ticketRequest.error, 5); + } + hide(); + this.setState({pending: false}); + }); + }; + + scrollToEditor = () => { + this.editorRef && + this.editorRef.scrollIntoView && + this.editorRef.scrollIntoView({behavior: 'smooth', block: 'end'}); + }; + + renderComments = () => { + const {editComment} = this.state; + return ( +
+ {[this.ticket, ...this.comments] + .filter(Boolean) + .filter((comment) => comment.description || comment.body) + .map((comment, index) => { + const commentId = comment.iid || comment.id; + if (editComment !== undefined && editComment === commentId) { + return ( + + ); + } + return ( + + ); + })} +
+ ); + }; + + renderInfoSection = () => { + const getLabel = (labels) => { + const [label] = (labels || []) + .filter(label => this.predefinedLabels.includes(label.toLowerCase())); + return label; + }; + return ( +
+
+ Author: + +
+
+ Status: +
+
+
+
+ ); + } + + render () { + if (!this.pending && !this.ticket) { + return ( + + ); + } + if (this.pending && !this.ticket) { + return ; + } + return ( +
+
+
+ {this.ticket.title} + + {`#${this.ticket.iid}`} + +
+
+
+ {this.renderComments()} +
{ + this.editorRef = el; + }} + > + +
+
+ {this.renderInfoSection()} +
+
+
+ ); + } +} diff --git a/client/src/components/special/tickets/special/ticket/ticket.css b/client/src/components/special/tickets/special/ticket/ticket.css new file mode 100644 index 0000000000..914d8cc4a9 --- /dev/null +++ b/client/src/components/special/tickets/special/ticket/ticket.css @@ -0,0 +1,94 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.container { + display: flex; + justify-content: center; +} + +.content { + display: flex; + flex-direction: column; + max-width: 1400px; + flex-grow: 1; +} + +.ticket-header { + margin: 0 0 5px 5px; + font-size: larger; + display: flex; + flex-direction: row; + align-items: center; +} + +.ticket-header .ticket-id { + opacity: 0.8; + font-size: larger; + margin-left: 10px; +} + +.comments-section { + display: flex; + flex-direction: column; + flex-grow: 1; +} + +.comments-container { + margin-left: 25px; + margin-bottom: 20px; + border-left-style: dashed !important; +} + +.comments-container .card { + margin-left: -25px; +} + +.comments-container .card:not(:last-child) { + margin-bottom: 25px; +} + +.info-section { + margin-left: 20px; + display: flex; + flex-direction: column; + min-width: 200px; +} + +.info-block { + display: flex; + flex-direction: column; + padding-left: 10px; +} + +.info-block.row { + flex-direction: row; +} + +.info-block:not(:last-child) { + margin-bottom: 10px; + padding-bottom: 12px; +} + +.info-block:last-child { + border-bottom: none !important; +} + +.labels-row { + display: flex; + flex-direction: row; + flex-wrap: wrap; + margin-left: 10px; +} diff --git a/client/src/components/special/tickets/special/tickets-list/index.js b/client/src/components/special/tickets/special/tickets-list/index.js new file mode 100644 index 0000000000..ee2203c546 --- /dev/null +++ b/client/src/components/special/tickets/special/tickets-list/index.js @@ -0,0 +1,398 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import PropTypes from 'prop-types'; +import classNames from 'classnames'; +import {inject, observer} from 'mobx-react'; +import {computed, observable} from 'mobx'; +import { + Button, + Select, + Dropdown, + Icon, + Pagination, + Menu, + Input, + Spin, + message, + Alert +} from 'antd'; +import displayDate from '../../../../../utils/displayDate'; +import highlightText from '../../../../special/highlightText'; +import Label from '../label'; +import GitlabIssuesLoad from '../../../../../models/gitlab-issues/GitlabIssuesLoad'; +import getAuthor from '../utilities/get-author'; +import UserName from '../../../UserName'; +import styles from './ticket-list.css'; + +const PAGE_SIZE = 20; + +@inject('preferences') +@observer +class TicketsList extends React.Component { + static propTypes = { + refreshTokenId: PropTypes.number, + onSelectTicket: PropTypes.func, + onDeleteTicket: PropTypes.func, + onNavigateBack: PropTypes.func, + pending: PropTypes.bool, + hideControls: PropTypes.bool + }; + + state = { + filters: { + search: '', + labels: [] + }, + page: 1, + pending: false + } + + _filtersRefreshTimeout; + + @observable + ticketsRequest; + + componentDidMount () { + this.fetchTickets(); + } + + componentDidUpdate (prevProps) { + if (this.props.refreshTokenId !== prevProps.refreshTokenId) { + this.fetchTickets(); + } + } + + componentWillUnmount () { + if (this._filtersRefreshTimeout) { + clearTimeout(this._filtersRefreshTimeout); + } + } + + get pending () { + return this.props.pending || this.state.pending; + } + + @computed + get tickets () { + if ( + this.ticketsRequest && + this.ticketsRequest.loaded && + this.ticketsRequest.value + ) { + return this.ticketsRequest.value.elements || []; + } + return []; + } + + @computed + get totalTickets () { + if ( + this.ticketsRequest && + this.ticketsRequest.loaded && + this.ticketsRequest.value + ) { + return this.ticketsRequest.value.totalCount || 0; + } + return 0; + } + + @computed + get predefinedLabels () { + const {preferences} = this.props; + if (preferences && preferences.loaded) { + return (preferences.gitlabIssueStatuses || []) + .map(status => status.toLowerCase()); + } + return []; + } + + get filtersTouched () { + const {filters} = this.state; + return filters.search || filters.labels.length > 0; + } + + onFiltersChange = (field, eventType) => event => { + let value; + switch (eventType) { + case 'checkbox': + value = event.target.checked; + break; + case 'input': + value = event.target.value; + break; + case 'select': + value = event; + break; + default: + value = undefined; + } + this.setState({ + page: 1, + filters: { + ...this.state.filters, + [field]: value + } + }, () => { + if (this._filtersRefreshTimeout) { + clearTimeout(this._filtersRefreshTimeout); + } + if (value !== undefined) { + this._filtersRefreshTimeout = setTimeout(() => { + this.fetchTickets(); + }, 600); + } + }); + }; + + clearFilters = () => { + this.setState({ + page: 1, + filters: { + search: '', + labels: [] + } + }, () => { + this.fetchTickets(); + }); + }; + + fetchTickets = (page = 1) => { + this.setState({ + page, + pending: true + }, async () => { + const {filters} = this.state; + this.ticketsRequest = new GitlabIssuesLoad(page, PAGE_SIZE); + await this.ticketsRequest.send(filters); + if (this.ticketsRequest.error) { + message.error(this.ticketsRequest.error, 5); + } + this.setState({pending: false}); + }); + }; + + onPageChange = page => { + this.setState({page}, () => { + const {page} = this.state; + this.fetchTickets(page); + }); + }; + + onSelectMenu = (key, ticket) => { + const {onDeleteTicket} = this.props; + if (key === 'delete') { + onDeleteTicket && onDeleteTicket(ticket.iid); + } + }; + + onSelectTicket = (id) => { + const {onSelectTicket} = this.props; + if (this.pending || id === undefined) { + return null; + } + onSelectTicket && onSelectTicket(id); + }; + + renderHeader = () => { + const {filters} = this.state; + return ( +
+ + Status + + + Title + +
+ + + +
+
+ ); + }; + + renderTicket = (ticket) => { + const {hideControls} = this.props; + const {filters} = this.state; + const menu = ( + this.onSelectMenu(key, ticket)} + selectedKeys={[]} + style={{cursor: 'pointer'}} + > + + Close ticket + + + Delete + + + ); + const getLabel = (labels) => { + const [label] = (labels || []) + .filter(label => this.predefinedLabels.includes(label.toLowerCase())); + return label; + }; + return ( +
this.onSelectTicket(ticket.iid)} + > +
+ ); + } + + render () { + const {page} = this.state; + if (this.ticketsRequest && this.ticketsRequest.error) { + return ( + + ); + } + return ( +
+
+ {this.renderHeader()} + + {this.tickets.map(this.renderTicket)} + +
+
+ +
+
+ ); + } +} + +export default TicketsList; diff --git a/client/src/components/special/tickets/special/tickets-list/ticket-list.css b/client/src/components/special/tickets/special/tickets-list/ticket-list.css new file mode 100644 index 0000000000..77f67891d0 --- /dev/null +++ b/client/src/components/special/tickets/special/tickets-list/ticket-list.css @@ -0,0 +1,114 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.container { + display: flex; + flex-direction: column; + overflow: hidden; + border-radius: 3px; + padding: 2px 0 0; +} + +.ticket-container { + display: grid; + grid: "status title controls" auto / 90px 1fr auto; + padding: 3px 0; + cursor: pointer; + transition: all 0.2s ease; +} + +.ticket-container.header { + cursor: default; + position: sticky; + top: 0; + z-index: 2; +} + +.ticket-container .status { + grid-area: status; + display: flex; + align-items: center; + font-size: larger; + padding: 0 5px; + justify-content: flex-start; +} + +.ticket-container:not(.header) .status { + height: 24px; + margin: auto; + font-size: small; + justify-content: center; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 100%; + display: inline-block; +} + +.ticket-container .title { + grid-area: title; + display: flex; + flex-direction: column; + padding: 0 10px; +} + +.ticket-container .controls { + grid-area: controls; + display: flex; + align-items: center; + justify-content: flex-end; +} + +.tickets-table { + position: relative; + overflow-y: auto; + overflow-x: hidden; +} + +.table-control { + margin-right: 5px; +} + +.table-control.clear-button { + width: 25px; + display: flex; + justify-content: center; + align-items: center; + font-size: larger; + transition: all 0.2s ease; +} + +.table-control.clear-button.hidden { + width: 0; + height: 0; + margin: 0; + padding: 0; + border: 0; + overflow: hidden; + transform: scale(0); +} + +.table-spin { + min-height: 35px; +} + +.pagination-row { + display: flex; + flex-wrap: nowrap; + align-items: center; + justify-content: flex-end; + padding: 10px 0; +} diff --git a/client/src/components/special/tickets/special/utilities/blob-files-to-base64.js b/client/src/components/special/tickets/special/utilities/blob-files-to-base64.js new file mode 100644 index 0000000000..c0f597f471 --- /dev/null +++ b/client/src/components/special/tickets/special/utilities/blob-files-to-base64.js @@ -0,0 +1,32 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default async function blobFilesToBase64 (files = []) { + const promises = files.map(file => { + return new Promise((resolve) => { + try { + const reader = new FileReader(); + reader.onloadend = () => resolve({[file.name]: reader.result}); + reader.readAsDataURL(file); + } catch { + resolve(null); + } + }); + }); + // todo: handle name collisions (object keys) ? + return Promise.all(promises) + .then(results => results.filter(Boolean).reduce((a, c) => ({...a, ...c}))); +} diff --git a/client/src/components/special/tickets/special/utilities/get-author.js b/client/src/components/special/tickets/special/utilities/get-author.js new file mode 100644 index 0000000000..8f18aa347c --- /dev/null +++ b/client/src/components/special/tickets/special/utilities/get-author.js @@ -0,0 +1,31 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default function getAuthor (ticket = {}) { + const { + labels = [], + author + } = ticket; + const authorLabel = (labels || []) + .find(label => label.toLowerCase().includes('on behalf of')); + if (authorLabel) { + return authorLabel.split('of').pop().trim(); + } + if (author && author.name) { + return author.name; + } + return ''; +}; diff --git a/client/src/components/special/tickets/tickets-browser.css b/client/src/components/special/tickets/tickets-browser.css new file mode 100644 index 0000000000..9440e252bb --- /dev/null +++ b/client/src/components/special/tickets/tickets-browser.css @@ -0,0 +1,44 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.container { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + padding: 5px; + overflow-y: auto; +} + +.header-container { + display: flex; + flex-wrap: nowrap; + justify-content: flex-end; + align-items: center; + padding: 0 5px 5px; +} + +.header-container .heading { + margin-right: auto; + font-size: larger; +} + +.go-back-button { + margin-right: 10px; + display: flex; + align-items: center; + justify-content: center; +} diff --git a/client/src/models/gitlab-issues/GitlabIssueComment.js b/client/src/models/gitlab-issues/GitlabIssueComment.js new file mode 100644 index 0000000000..7bdfa91b6b --- /dev/null +++ b/client/src/models/gitlab-issues/GitlabIssueComment.js @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import RemotePost from '../basic/RemotePost'; + +export default class GitlabIssueComment extends RemotePost { + constructor (issueId) { + super(); + this.url = `/issue/gitlab/${issueId}/comment`; + } +} diff --git a/client/src/models/gitlab-issues/GitlabIssueCreate.js b/client/src/models/gitlab-issues/GitlabIssueCreate.js new file mode 100644 index 0000000000..e092808bab --- /dev/null +++ b/client/src/models/gitlab-issues/GitlabIssueCreate.js @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import RemotePost from '../basic/RemotePost'; + +export default class GitlabIssueCreate extends RemotePost { + constructor () { + super(); + this.url = '/issue/gitlab'; + } +} diff --git a/client/src/models/gitlab-issues/GitlabIssueDelete.js b/client/src/models/gitlab-issues/GitlabIssueDelete.js new file mode 100644 index 0000000000..0e84a470bf --- /dev/null +++ b/client/src/models/gitlab-issues/GitlabIssueDelete.js @@ -0,0 +1,32 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import RemotePost from '../basic/RemotePost'; + +export default class GitlabIssueDelete extends RemotePost { + constructor (issueId) { + super(); + this.constructor.fetchOptions = { + headers: { + 'Content-type': 'application/json; charset=UTF-8' + }, + mode: 'cors', + credentials: 'include', + method: 'DELETE' + }; + this.url = `/issue/gitlab/${issueId}`; + } +} diff --git a/client/src/models/gitlab-issues/GitlabIssueLoad.js b/client/src/models/gitlab-issues/GitlabIssueLoad.js new file mode 100644 index 0000000000..734ff4d0f5 --- /dev/null +++ b/client/src/models/gitlab-issues/GitlabIssueLoad.js @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Remote from '../basic/Remote'; + +export default class GitlabIssueLoad extends Remote { + constructor (issueId) { + super(); + this.url = `/issue/gitlab/${issueId}`; + } +} diff --git a/client/src/models/gitlab-issues/GitlabIssuesLoad.js b/client/src/models/gitlab-issues/GitlabIssuesLoad.js new file mode 100644 index 0000000000..dbb3b97403 --- /dev/null +++ b/client/src/models/gitlab-issues/GitlabIssuesLoad.js @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import RemotePost from '../basic/RemotePost'; + +export default class GitlabIssuesLoad extends RemotePost { + constructor (page = 1, pageSize = 10) { + super(); + this.url = `/issue/gitlab/filter?page=${page}&pageSize=${pageSize}`; + } +} diff --git a/client/src/models/pipelines/RunCount.js b/client/src/models/pipelines/RunCount.js index 612fb2325a..46ddc67f58 100644 --- a/client/src/models/pipelines/RunCount.js +++ b/client/src/models/pipelines/RunCount.js @@ -18,6 +18,7 @@ import {action, computed, observable} from 'mobx'; import RemotePost from '../basic/RemotePost'; import preferencesLoad from '../preferences/PreferencesLoad'; import continuousFetch from '../../utils/continuous-fetch'; +import {filtersAreEqual} from './pipeline-runs-filter'; const DEFAULT_STATUSES = [ 'RUNNING', @@ -26,6 +27,18 @@ const DEFAULT_STATUSES = [ 'RESUMING' ]; +const ALL_STATUSES = [ + 'RUNNING', + 'PAUSED', + 'PAUSING', + 'RESUMING', + 'STOPPED', + 'FAILURE', + 'SUCCESS' +]; + +export {ALL_STATUSES}; + class UserRunCount extends RemotePost { static fetchOptions = { headers: { @@ -59,6 +72,8 @@ class RunCount extends RemotePost { @observable usePreferenceValue = false; @observable onlyMasterJobs = true; @observable statuses = DEFAULT_STATUSES; + @observable pipelineIds = []; + @observable parentId; @observable _runsCount = 0; @@ -70,6 +85,8 @@ class RunCount extends RemotePost { * @property {string[]} [statuses] * @property {boolean} [onlyMasterJobs=true] * @property {boolean} [autoUpdate=false] + * @property {number[]} [pipelineIds=[]] + * @property {number|string} [parentId] */ /** @@ -82,13 +99,20 @@ class RunCount extends RemotePost { usePreferenceValue, statuses = DEFAULT_STATUSES, onlyMasterJobs = true, - autoUpdate + autoUpdate, + pipelineIds = [], + parentId } = options || {}; this.statuses = statuses; this.onlyMasterJobs = onlyMasterJobs; this.usePreferenceValue = usePreferenceValue; + this.pipelineIds = pipelineIds; + this.parentId = parentId; if (autoUpdate) { - continuousFetch({request: this}); + continuousFetch({ + request: this, + intervalMS: 10000 + }); } } @@ -103,21 +127,13 @@ class RunCount extends RemotePost { @computed get isDefault () { - try { - const currentStatuses = new Set(this.statuses); - if (currentStatuses.size !== DEFAULT_STATUSES.length) { - return false; + return filtersAreEqual( + this, + { + statuses: DEFAULT_STATUSES, + onlyMasterJobs: true } - for (const status of DEFAULT_STATUSES) { - if (!currentStatuses.has(status)) { - return false; - } - } - return this.onlyMasterJobs; - } catch (_) { - // empty - } - return true; + ); } /** @@ -128,21 +144,7 @@ class RunCount extends RemotePost { if (!otherRequest) { return false; } - try { - const currentStatuses = new Set(this.statuses); - if (currentStatuses.size !== otherRequest.statuses.length) { - return false; - } - for (const status of otherRequest.statuses) { - if (!currentStatuses.has(status)) { - return false; - } - } - return this.onlyMasterJobs === otherRequest.onlyMasterJobs; - } catch (_) { - // empty - } - return true; + return filtersAreEqual(this, otherRequest); } @computed @@ -163,6 +165,8 @@ class RunCount extends RemotePost { await super.send({ statuses: this.statuses || ['RUNNING', 'PAUSED', 'PAUSING', 'RESUMING'], userModified: !this.onlyMasterJobs, + parentId: this.parentId, + pipelineIds: this.pipelineIds, eagerGrouping: false }); this._runsCount = this.value; diff --git a/client/src/components/runs/run-table/filter.js b/client/src/models/pipelines/pipeline-runs-filter.js similarity index 86% rename from client/src/components/runs/run-table/filter.js rename to client/src/models/pipelines/pipeline-runs-filter.js index beb2c304d1..5cbc7038da 100644 --- a/client/src/components/runs/run-table/filter.js +++ b/client/src/models/pipelines/pipeline-runs-filter.js @@ -1,19 +1,3 @@ -/* - * Copyright 2017-2022 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - import moment from 'moment-timezone'; function asStringArray (array) { diff --git a/client/src/models/preferences/PreferencesLoad.js b/client/src/models/preferences/PreferencesLoad.js index 5d80aa159c..9f59071c41 100644 --- a/client/src/models/preferences/PreferencesLoad.js +++ b/client/src/models/preferences/PreferencesLoad.js @@ -259,6 +259,19 @@ class PreferencesLoad extends Remote { return 1; } + @computed + get gitlabIssueStatuses () { + const value = this.getPreferenceValue('git.gitlab.issue.statuses'); + if (value) { + try { + return JSON.parse(value); + } catch (e) { + console.warn('Error parsing "git.gitlab.issue.statuses" preference:', e); + } + } + return []; + } + @computed get sharedStoragesSystemDirectory () { const value = this.getPreferenceValue('data.sharing.storage.folders.directory'); diff --git a/client/src/themes/styles/layout.less b/client/src/themes/styles/layout.less index afc6ff920f..2041471772 100644 --- a/client/src/themes/styles/layout.less +++ b/client/src/themes/styles/layout.less @@ -137,6 +137,10 @@ a.cp-danger:focus, color: @primary-color; } +.cp-primary.border { + border-color: @primary-color; +} + .cp-disabled { color: @application-color-disabled; } @@ -149,10 +153,18 @@ a.cp-danger:focus, color: @color-yellow; } +.cp-warning.border { + border-color: @color-yellow; +} + .cp-success { color: @color-green; } +.cp-success.border { + border-color: @color-green; +} + .cp-error, .cp-danger { color: @color-red; diff --git a/client/src/themes/utilities/theme.less.template.js b/client/src/themes/utilities/theme.less.template.js index 97da366f8b..9558fde591 100644 --- a/client/src/themes/utilities/theme.less.template.js +++ b/client/src/themes/utilities/theme.less.template.js @@ -210,6 +210,9 @@ export default ` @THEME .cp-primary { color: @primary-color; } +@THEME .cp-primary.border { + border-color: @primary-color; +} @THEME .cp-disabled { color: @application-color-disabled; } @@ -219,9 +222,15 @@ export default ` @THEME .cp-warning { color: @color-yellow; } +@THEME .cp-warning.border { + border-color: @color-yellow; +} @THEME .cp-success { color: @color-green; } +@THEME .cp-success.border { + border-color: @color-green; +} @THEME .cp-error, @THEME .cp-danger { color: @color-red; diff --git a/client/src/utils/filter/inputControl/FilterInput.js b/client/src/utils/filter/inputControl/FilterInput.js index eb34d63fc4..f29922f63b 100644 --- a/client/src/utils/filter/inputControl/FilterInput.js +++ b/client/src/utils/filter/inputControl/FilterInput.js @@ -55,7 +55,6 @@ import 'codemirror/addon/edit/trailingspace'; @observer export default class FilterInput extends React.Component { - static propTypes = { className: PropTypes.string, defaultValue: PropTypes.string, @@ -78,13 +77,16 @@ export default class FilterInput extends React.Component { } _onCodeChange = (editor, change) => { - if (this.props.onEdit && change.from) { - this.props.onEdit(editor.getValue(), change.from.ch + (change.from.sticky === 'after' ? -1 : 0)); + if (this.props.onEdit && change.from && editor) { + this.props.onEdit( + editor.getValue(), + change.from.ch + (change.from.sticky === 'after' ? -1 : 0) + ); } }; _onEnter = () => { - if (this.props.onEnter) { + if (this.props.onEnter && this.codeMirrorInstance) { this.props.onEnter(this.codeMirrorInstance.getValue()); } }; @@ -218,6 +220,7 @@ export default class FilterInput extends React.Component { token: ['variable-2'] }, { + // eslint-disable-next-line max-len regex: /(\s)*(<|<=|=|!=|>=|>)(\s)*('(?:[^\\]|\\.)*?(?:'|$)|"(?:[^\\]|\\.)*?(?:"|$)|[^\s'"()[\]{}/\\]+)/, token: [null, null, null, 'comment'] } diff --git a/core/src/main/java/com/epam/pipeline/entity/configuration/PipelineConfiguration.java b/core/src/main/java/com/epam/pipeline/entity/configuration/PipelineConfiguration.java index 5cd6e90cc3..5a7dbe2f47 100644 --- a/core/src/main/java/com/epam/pipeline/entity/configuration/PipelineConfiguration.java +++ b/core/src/main/java/com/epam/pipeline/entity/configuration/PipelineConfiguration.java @@ -20,6 +20,7 @@ import com.epam.pipeline.entity.git.GitCredentials; import com.epam.pipeline.entity.pipeline.run.ExecutionPreferences; import com.epam.pipeline.entity.pipeline.run.PipelineStartNotificationRequest; +import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy; import com.epam.pipeline.entity.pipeline.run.parameter.RunSid; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -31,6 +32,7 @@ import org.apache.commons.collections4.ListUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -47,7 +49,7 @@ @Getter @NoArgsConstructor @EqualsAndHashCode -public class PipelineConfiguration { +public class PipelineConfiguration implements Cloneable { private static final String MAIN_FILE = "main_file"; private static final String MAIN_CLASS = "main_class"; @@ -159,6 +161,10 @@ public class PipelineConfiguration { private Map kubeLabels; + private RunAssignPolicy podAssignPolicy; + + private String kubeServiceAccount; + @JsonProperty(value = RAW_EDIT) private Boolean rawEdit; @@ -221,4 +227,36 @@ private List adjustPrincipal(final List runsSids, final boolean .peek(runSid -> runSid.setIsPrincipal(principal)) .collect(Collectors.toList()); } + + @JsonIgnore + @Override + public PipelineConfiguration clone() { + try { + final PipelineConfiguration clone = (PipelineConfiguration) super.clone(); + if (this.parameters != null) { + clone.setParameters(new HashMap<>(this.parameters)); + } + if (this.environmentParams != null) { + clone.setEnvironmentParams(new HashMap<>(this.environmentParams)); + } + if (this.sharedWithUsers != null) { + clone.setSharedWithUsers(new ArrayList<>(this.sharedWithUsers)); + } + if (this.sharedWithRoles != null) { + clone.setSharedWithRoles(new ArrayList<>(this.sharedWithRoles)); + } + if (this.notifications != null) { + clone.setNotifications(new ArrayList<>(this.notifications)); + } + if (this.tags != null) { + clone.setTags(new HashMap<>(this.tags)); + } + if (this.kubeLabels != null) { + clone.setKubeLabels(new HashMap<>(this.kubeLabels)); + } + return clone; + } catch (CloneNotSupportedException e) { + throw new AssertionError("There was an error while trying to clone PipelineConfiguration object", e); + } + } } diff --git a/core/src/main/java/com/epam/pipeline/entity/pipeline/run/PipelineStart.java b/core/src/main/java/com/epam/pipeline/entity/pipeline/run/PipelineStart.java index fa0da74ef7..76668351d4 100644 --- a/core/src/main/java/com/epam/pipeline/entity/pipeline/run/PipelineStart.java +++ b/core/src/main/java/com/epam/pipeline/entity/pipeline/run/PipelineStart.java @@ -43,7 +43,9 @@ public class PipelineStart { private String configurationName; private Integer nodeCount; private String workerCmd; + // TODO: Should be relatively easy to switch to runAssignPolicy in the feature private Long parentRunId; + private RunAssignPolicy podAssignPolicy; private Boolean isSpot; private List runSids; private Long cloudRegionId; @@ -54,6 +56,7 @@ public class PipelineStart { private String runAs; private List notifications; private Map kubeLabels; + private String kubeServiceAccount; @JsonDeserialize(using = PipelineConfValuesMapDeserializer.class) private Map params; diff --git a/core/src/main/java/com/epam/pipeline/entity/pipeline/run/RunAssignPolicy.java b/core/src/main/java/com/epam/pipeline/entity/pipeline/run/RunAssignPolicy.java new file mode 100644 index 0000000000..bca4b524b1 --- /dev/null +++ b/core/src/main/java/com/epam/pipeline/entity/pipeline/run/RunAssignPolicy.java @@ -0,0 +1,116 @@ +/* + * Copyright 2023 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.entity.pipeline.run; + +import lombok.Builder; +import lombok.ToString; +import lombok.Value; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + + +/** + * Object to describe assign strategy of a run pod to a node. + * @see RunAssignPolicy.PodAssignSelector and + * @see RunAssignPolicy.PodAssignTolerance for mode informations + * */ +@Value +@Builder +@ToString +public class RunAssignPolicy { + + PodAssignSelector selector; + List tolerances; + + public boolean isMatch(final String label, final String value) { + if (!isValid()) { + return false; + } + return this.selector.label.equals(label) && (value == null || this.selector.value.equals(value)); + } + + public boolean isMatch(final String label) { + return isMatch(label, null); + } + + public Optional ifMatchThenMapValue(final String label, Function caster) { + if (isMatch(label)) { + return Optional.of(caster.apply(selector.value)); + } + return Optional.empty(); + } + + public boolean isValid() { + if (selector == null) { + return false; + } + return StringUtils.isNotBlank(selector.label) && StringUtils.isNotBlank(selector.value); + } + + public Map getTolerances() { + return ListUtils.emptyIfNull(tolerances).stream() + .filter(t -> StringUtils.isNotBlank(t.label)) + .collect(Collectors.toMap(PodAssignTolerance::getLabel, PodAssignTolerance::getValue)); + } + + /** + * Object to implement selection capability while scheduling run pod. + + * User can specify selector as (with regard to real kube labels): label="node-role/cp-api-srv" value="true" + * to assign a run to a node with label node-role/cp-api-srv=true + * */ + @Value + @Builder + @ToString + public static class PodAssignSelector { + String label; + String value; + } + + /** + * Object to implement tolerance capability while scheduling run pod. + + * label="label-key", value="value" - equals to tolerance: + * tolerations: + * - key: "key1" + * operator: "Equal" + * value: "value" + * effect: "*" + + * label="label-key", value="" or value=null - equals to tolerance: + * tolerations: + * - key: "key1" + * operator: "Exists" + * effect: "*" + + * For more information see + * Taints, Tolerations + * */ + @Value + @Builder + @ToString + public static class PodAssignTolerance { + String label; + String value; + } +} diff --git a/deploy/contents/install/app/configure-utils.sh b/deploy/contents/install/app/configure-utils.sh index 0af70ac473..676c2c68c5 100644 --- a/deploy/contents/install/app/configure-utils.sh +++ b/deploy/contents/install/app/configure-utils.sh @@ -1419,6 +1419,49 @@ function api_register_data_transfer_pipeline { print_ok "Data transfer pipeline $CP_API_SRV_SYSTEM_TRANSFER_PIPELINE_FRIENDLY_NAME is registered with ID $pipeline_id and tag $dt_pipeline_version" } +function api_register_system_jobs_pipeline { + local sj_role_grant="${1:-ROLE_ADMIN}" + local sj_role_permissions="21" + local sj_pipeline_version=${CP_API_SRV_SYSTEM_JOBS_PIPELINE_VERSION:-v1} + + # 0. Verify and update config.json template + local sj_pipeline_dir="$OTHER_PACKAGES_PATH/system-jobs" + local sj_pipeline_config_json="$sj_pipeline_dir/config.json" + if [ ! -f "$sj_pipeline_config_json" ]; then + print_err "config.json is not found for the system jobs pipeline at ${sj_pipeline_config_json}. Pipeline will not be registered" + return 1 + fi + + local sj_pipeline_config_json_content="$(envsubst < "$sj_pipeline_config_json")" + cat <<< "$sj_pipeline_config_json_content" > "$sj_pipeline_config_json" + + # 1. Register a system jobs pipeline in general + api_register_pipeline "$CP_API_SRV_SYSTEM_FOLDER_NAME" \ + "$CP_API_SRV_SYSTEM_JOBS_PIPELINE_FRIENDLY_NAME" \ + "$CP_API_SRV_SYSTEM_JOBS_PIPELINE_DESCRIPTION" \ + "$sj_pipeline_dir" \ + "$sj_role_grant" \ + "$sj_role_permissions" \ + "$sj_pipeline_version" + + if [ $? -ne 0 ]; then + print_err "Error occurred while registering a system jobs pipeline (see any output above). API will not be configured to use system jobs" + return 1 + fi + + # 2. Get system jobs pipeline registered id + local pipeline_id="$(api_get_entity_id "$CP_API_SRV_SYSTEM_JOBS_PIPELINE_FRIENDLY_NAME" "pipeline")" + if [ $? -ne 0 ] || [ ! "$pipeline_id" ]; then + print_err "Unable to determine ID of the data system jobs pipeline. API will not be configured to use data system jobs pipeline" + return 1 + fi + + # 3. Register system jobs pipeline in the preferences + api_set_preference "system.jobs.pipeline.id" "$pipeline_id" "true" + + print_ok "System jobs pipeline $CP_API_SRV_SYSTEM_JOBS_PIPELINE_FRIENDLY_NAME is registered with ID $pipeline_id and tag $sj_pipeline_version" +} + function api_register_email_templates { CP_EMAIL_TEMPLATES_CONFIGS_PATH=${CP_EMAIL_TEMPLATES_CONFIGS_PATH:-"$INSTALL_SCRIPT_PATH/../email-templates/configs"} CP_EMAIL_TEMPLATES_CONTENTS_PATH=${CP_EMAIL_TEMPLATES_CONTENTS_PATH:-"$INSTALL_SCRIPT_PATH/../email-templates/contents"} diff --git a/deploy/contents/install/app/install.sh b/deploy/contents/install/app/install.sh index 42342bcbe1..88a3bb560e 100644 --- a/deploy/contents/install/app/install.sh +++ b/deploy/contents/install/app/install.sh @@ -939,6 +939,9 @@ if is_service_requested cp-git; then print_info "-> Registering DataTransfer pipeline" api_register_data_transfer_pipeline + print_info "-> Registering System Jobs pipeline" + api_register_system_jobs_pipeline + if [ "$CP_DEPLOY_DEMO" ]; then print_info "-> Registering Demo pipelines" api_register_demo_pipelines diff --git a/deploy/docker/build-dockers.sh b/deploy/docker/build-dockers.sh index 86ed8cb595..1752c5a64d 100644 --- a/deploy/docker/build-dockers.sh +++ b/deploy/docker/build-dockers.sh @@ -455,6 +455,13 @@ MD_TOOLS_DOCKERS_SOURCES_PATH=$DOCKERS_SOURCES_PATH/cp-tools/md # FIXME: Add gromacs and namd +######################## +# System dockers +######################## + +SYSTEM_TOOLS_DOCKERS_SOURCES_PATH=$DOCKERS_SOURCES_PATH/cp-tools/system + +build_and_push_tool $SYSTEM_TOOLS_DOCKERS_SOURCES_PATH/system-job "$CP_DIST_REPO_NAME:tools-system-system-job-${DOCKERS_VERSION}" "system/system-job-launcher:latest" ######################## # E2E tests dockers diff --git a/deploy/docker/cp-tools/system/system-job/Dockerfile b/deploy/docker/cp-tools/system/system-job/Dockerfile new file mode 100644 index 0000000000..f7860234dd --- /dev/null +++ b/deploy/docker/cp-tools/system/system-job/Dockerfile @@ -0,0 +1,80 @@ +FROM library/centos:7 + + +ENV CP_PIP_EXTRA_ARGS="--index-url http://cloud-pipeline-oss-builds.s3-website-us-east-1.amazonaws.com/tools/python/pypi/simple --trusted-host cloud-pipeline-oss-builds.s3-website-us-east-1.amazonaws.com" +ENV COMMON_REPO_DIR=/usr/sbin/CommonRepo +ARG CP_API_DIST_URL + +# Configure cloud-pipeline yum repository +RUN curl -sk "https://cloud-pipeline-oss-builds.s3.amazonaws.com/tools/repos/centos/7/cloud-pipeline.repo" > /etc/yum.repos.d/cloud-pipeline.repo && \ + yum --disablerepo=* --enablerepo=cloud-pipeline install yum-priorities -y && \ + yum-config-manager --save --setopt=\*.skip_if_unavailable=true && \ + sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf && \ + sed -i 's/^#baseurl=/baseurl=/g' /etc/yum.repos.d/*.repo && \ + sed -i 's/^metalink=/#metalink=/g' /etc/yum.repos.d/*.repo && \ + sed -i 's/^mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/*.repo + +# Install common dependencies +RUN yum install -y curl \ + wget \ + make \ + unzip \ + nano \ + git \ + python \ + fuse \ + tzdata \ + acl \ + coreutils \ + openssh-server \ + yum-utils && \ + yum clean all + +# Install pip +RUN curl -s https://cloud-pipeline-oss-builds.s3.amazonaws.com/tools/pip/2.7/get-pip.py | python2 && \ + python2 -m pip install $CP_PIP_EXTRA_ARGS -I -q setuptools==44.1.1 + +# Install "pipe" python package +RUN if [ "$CP_API_URL" ]; then \ + mkdir -p /tmp/cp && \ + curl -s -k "$CP_API_URL"/pipe.tar.gz > /tmp/cp/pipe.tar.gz && \ + cd /tmp/cp && \ + tar -zxf /tmp/cp/pipe.tar.gz -C /tmp/ && \ + mv /tmp/pipe /usr/lib/ && \ + chmod +rx -R /usr/lib/pipe && \ + ln -s /usr/lib/pipe ${CP_USR_BIN}/pipe && \ + ln -s ${CP_USR_BIN}/pipe/pipe /usr/bin/pipe && \ + rm -rf /tmp/pipe.tar.gz && \ + rm -rf /tmp/cp; \ + fi + +# Install Lustre client +RUN cd /tmp && \ + wget -q https://cloud-pipeline-oss-builds.s3.amazonaws.com/tools/lustre/client/rpm/lustre-client-2.12.5-1.el7.x86_64.tar.gz -O lustre-client.tar.gz && \ + mkdir -p lustre-client && \ + tar -xzvf lustre-client.tar.gz -C lustre-client/ && \ + rpm -i --justdb --quiet --nodeps --force lustre-client/dependencies/*.rpm && \ + yum install -y lustre-client/*.rpm && \ + package-cleanup --cleandupes -y && \ + rm -rf lustre-client* && \ + yum clean all + +RUN curl -LO "https://dl.k8s.io/release/v1.15.4/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mv kubectl /usr/bin + +# AWS CLI +RUN cd /opt/ && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && ./aws/install \ + && rm -rf awscliv2.zip && rm -rf ./aws/ + +# Azure CLI +RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + sh -c 'echo -e "[azure-cli]"' >> /etc/yum.repos.d/azure-cli.repo && \ + sh -c 'echo -e "name=Azure CLI"' >> /etc/yum.repos.d/azure-cli.repo && \ + sh -c 'echo -e "baseurl=https://packages.microsoft.com/yumrepos/azure-cli"' >> /etc/yum.repos.d/azure-cli.repo && \ + sh -c 'echo -e "enabled=1"' >> /etc/yum.repos.d/azure-cli.repo && \ + sh -c 'echo -e "gpgcheck=1"' >> /etc/yum.repos.d/azure-cli.repo && \ + sh -c 'echo -e "gpgkey=https://packages.microsoft.com/keys/microsoft.asc"' >> /etc/yum.repos.d/azure-cli.repo && \ + yum install -y azure-cli && yum clean all diff --git a/deploy/docker/cp-tools/system/system-job/README.md b/deploy/docker/cp-tools/system/system-job/README.md new file mode 100644 index 0000000000..1250cb4536 --- /dev/null +++ b/deploy/docker/cp-tools/system/system-job/README.md @@ -0,0 +1,9 @@ +# System Jobs Image + +Image to be used with Cloud Pipeline System Jobs. +It includes packages pre-installed such as: + - System packages (curl, nano, git, etc) + - Kubectl + - Pipe CLI + - Cloud CLI (AWS/Azure/GCP) + - LustreFS client \ No newline at end of file diff --git a/deploy/docker/cp-tools/system/system-job/spec.json b/deploy/docker/cp-tools/system/system-job/spec.json new file mode 100644 index 0000000000..0d1763dd5d --- /dev/null +++ b/deploy/docker/cp-tools/system/system-job/spec.json @@ -0,0 +1,6 @@ +{ + "short_description": "Image to use to run System Jobs on Cloud-Pipeline", + "instance_type": "${CP_PREF_CLUSTER_INSTANCE_TYPE}", + "disk_size": "50", + "default_command": "sleep infinity" +} diff --git a/workflows/pipe-common/pipeline/cluster/cluster.py b/workflows/pipe-common/pipeline/cluster/cluster.py index 439c494d5a..6799f102e4 100644 --- a/workflows/pipe-common/pipeline/cluster/cluster.py +++ b/workflows/pipe-common/pipeline/cluster/cluster.py @@ -112,7 +112,7 @@ def _parse_job_exit_status(output): return None for line in output.splitlines(): line = line.strip() - if line.strip().startswith('exit_status'): + if line.startswith('exit_status'): parts = line.split() if len(parts) != 2: return None diff --git a/workflows/pipe-common/pipeline/cluster/utils.py b/workflows/pipe-common/pipeline/cluster/utils.py index 24a229bd35..b0f118b731 100644 --- a/workflows/pipe-common/pipeline/cluster/utils.py +++ b/workflows/pipe-common/pipeline/cluster/utils.py @@ -43,9 +43,11 @@ def create_directory(path, name, lock): def run(job_command, get_output=True, env=None): if get_output: - process = subprocess.Popen(job_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + process = subprocess.Popen(job_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, + universal_newlines=True) else: - process = subprocess.Popen(job_command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=env) + process = subprocess.Popen(job_command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, env=env, + universal_newlines=True) stdout, stderr = process.communicate() exit_code = process.wait() return stdout, stderr, exit_code diff --git a/workflows/pipe-common/scripts/autoscale_sge.py b/workflows/pipe-common/scripts/autoscale_sge.py index 35dcc76583..0c1b94bbb7 100644 --- a/workflows/pipe-common/scripts/autoscale_sge.py +++ b/workflows/pipe-common/scripts/autoscale_sge.py @@ -1717,12 +1717,11 @@ def record(self, run_id): try: Logger.info('Recording details of additional worker #%s...' % run_id) run = self._api.load_run(run_id) - initialize_node_tasks = self._api.load_task(run_id, 'InitializeNode') run_name = run.get('podId') run_started = self._to_datetime(run.get('startDate')) run_stopped = self._to_datetime(run.get('endDate')) instance_type = run.get('instance', {}).get('nodeType') - has_insufficient_instance_capacity = self._has_insufficient_instance_capacity(run, initialize_node_tasks) + has_insufficient_instance_capacity = self._has_insufficient_instance_capacity(run) record = GridEngineWorkerRecord(id=run_id, name=run_name, instance_type=instance_type, started=run_started, stopped=run_stopped, has_insufficient_instance_capacity=has_insufficient_instance_capacity) @@ -1740,14 +1739,13 @@ def _to_datetime(self, run_started): return None return datetime.strptime(run_started, self._datetime_format) - def _has_insufficient_instance_capacity(self, run, initialize_node_tasks): - if initialize_node_tasks: - initialize_node_task = initialize_node_tasks[-1] - if initialize_node_task.get('status') == 'FAILURE' \ - and 'InsufficientInstanceCapacity' in initialize_node_task.get('logText', ''): - Logger.warn('Insufficient instance capacity detected for %s instance type' - % run.get('instance', {}).get('nodeType')) - return True + def _has_insufficient_instance_capacity(self, run): + run_status = run.get('status') + run_status_reason = run.get('stateReasonMessage') + if run_status == 'FAILURE' and run_status_reason == 'Insufficient instance capacity.': + Logger.warn('Insufficient instance capacity detected for %s instance type' + % run.get('instance', {}).get('nodeType')) + return True return False def get(self): diff --git a/workflows/pipe-templates/__SYSTEM/system_jobs/config.json b/workflows/pipe-templates/__SYSTEM/system_jobs/config.json new file mode 100755 index 0000000000..bd3a30cc8d --- /dev/null +++ b/workflows/pipe-templates/__SYSTEM/system_jobs/config.json @@ -0,0 +1,68 @@ +[ + { + "name" : "default", + "description" : "System Jobs launch command configuration", + "configuration" : { + "main_file" : "launch_system_job.sh", + "instance_size" : "${CP_PREF_CLUSTER_INSTANCE_TYPE}", + "instance_disk" : "50", + "docker_image" : "system/system-job-launcher:latest", + "timeout" : 0, + "cmd_template" : "bash ${CP_DOLLAR}SCRIPTS_DIR/src/[main_file]", + "language" : "bash", + "kubeServiceAccount": "default", + "podAssignPolicy": { + "selector": { + "label": "cloud-pipeline/cp-api-srv", + "value": "true" + }, + "tolerances": [ + { + "label": "node-role.kubernetes.io/master", + "value": "" + } + ] + }, + "parameters" : { + "CP_SYSTEM_JOBS_RESULTS" : { + "value" : "${CP_PREF_SYSTEM_JOB_STORAGE_RESULT_FOLDER}", + "type" : "output", + "required" : false, + "no_override" : false + }, + "CP_SYSTEM_JOB_SYSTEM_FS" : { + "value" : "", + "type" : "string", + "required" : false + }, + "CP_SYSTEM_SCRIPTS_LOCATION" : { + "type" : "string", + "required" : true, + "no_override" : false + }, + "CP_SYSTEM_JOB" : { + "type" : "string", + "required" : true, + "no_override" : false + }, + "CP_SYSTEM_JOB_PARAMS" : { + "type" : "string", + "required" : false, + "no_override" : false + }, + "CP_SYSTEM_JOBS_OUTPUT_TASK" : { + "type" : "string", + "required" : false, + "no_override" : false + }, + "CP_CAP_LIMIT_MOUNTS" : { + "value" : "None", + "type" : "string", + "required" : false + } + }, + "is_spot" : true + }, + "default" : true + } +] diff --git a/workflows/pipe-templates/__SYSTEM/system_jobs/src/launch_system_job.sh b/workflows/pipe-templates/__SYSTEM/system_jobs/src/launch_system_job.sh new file mode 100644 index 0000000000..c9e80e7931 --- /dev/null +++ b/workflows/pipe-templates/__SYSTEM/system_jobs/src/launch_system_job.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +configure_cloud_credentials() { + + pipe_log_info "Configuring cloud credentials" $INIT_TASK_NAME + pipe_log_info "Provider is defined as $CLOUD_PROVIDER" $INIT_TASK_NAME + + if [ "$CLOUD_PROVIDER" == "AWS" ]; then + export AWS_DEFAULT_REGION="$CLOUD_REGION" + export AWS_ACCESS_KEY_ID=`eval echo "$"{CP_ACCOUNT_ID_${CLOUD_REGION_ID}}` + export AWS_SECRET_ACCESS_KEY=`eval echo "$"{CP_ACCOUNT_KEY_${CLOUD_REGION_ID}}` + export AWS_SESSION_TOKEN=`eval echo "$"{CP_ACCOUNT_TOKEN_${CLOUD_REGION_ID}}` + elif [ "$CLOUD_PROVIDER" == "AZURE" ]; then + # Untested + export AZURE_STORAGE_ACCOUNT=`eval echo "$"{CP_ACCOUNT_ID_${CLOUD_REGION_ID}}` + export AZURE_STORAGE_ACCESS_KEY=`eval echo "$"{CP_ACCOUNT_KEY_${CLOUD_REGION_ID}}` + elif [ "$CLOUD_PROVIDER" == "GCP" ]; then + # Untested + export GOOGLE_APPLICATION_CREDENTIALS="root/.gcp_credentials.json" + echo `eval echo "$"{CP_CREDENTIALS_FILE_CONTENT_${CLOUD_REGION_ID}}` > $GOOGLE_APPLICATION_CREDENTIALS + else + pipe_log_warn "Cloud provider wasn't provided or unsupported: $CLOUD_PROVIDER. Skipping credentials configuration." $INIT_TASK_NAME + return + fi + pipe_log_info "$CLOUD_PROVIDER credentials successfully configured." $INIT_TASK_NAME +} + +mount_system_fs() { + + if [ -z "$CP_SYSTEM_JOB_SYSTEM_FS" ]; then + pipe_log_warn "CP_SYSTEM_JOB_SYSTEM_FS wasn't provided. Skipping mounting system FS." $INIT_TASK_NAME + return + fi + + if [ "$CLOUD_PROVIDER" == "AWS" ]; then + export CP_SYS_FS_MOUNT_LOCATION="/opt/cp_sys_fs" + mkdir -p $CP_SYS_FS_MOUNT_LOCATION + mount -t lustre -o noatime,flock $CP_SYSTEM_JOB_SYSTEM_FS $CP_SYS_FS_MOUNT_LOCATION + if [ $? -eq 0 ]; then + pipe_log_info "System FS: $CP_SYSTEM_JOB_SYSTEM_FS successfully mounted to: $CP_SYS_FS_MOUNT_LOCATION. And location is available in CP_SYS_FS_MOUNT_LOCATION." $INIT_TASK_NAME + else + pipe_log_warn "Problems with mounting System FS $CP_SYSTEM_JOB_SYSTEM_FS. Skipping." $INIT_TASK_NAME + fi + else + pipe_log_warn "Cloud provider wasn't provided or unsupported: $CLOUD_PROVIDER. Skipping mounting system FS." $INIT_TASK_NAME + fi + +} + +export INIT_TASK_NAME="ConfigureSystemJob" +export MAIN_TASK_NAME="${$CP_SYSTEM_JOBS_OUTPUT_TASK:-SystemJob}" + +configure_cloud_credentials +mount_system_fs + +# Just granting appropriate permissions +chmod -R +x $SCRIPTS_DIR/$CP_SYSTEM_SCRIPTS_LOCATION/ + +# Validation of required params +if [ -z "$CP_SYSTEM_SCRIPTS_LOCATION" ]; then + pipe_log_fail "CP_SYSTEM_SCRIPTS_LOCATION wasn't provided, exiting." $INIT_TASK_NAME +fi + +if [ -z "$CP_SYSTEM_JOB" ]; then + pipe_log_fail "CP_SYSTEM_JOB wasn't provided, exiting." $INIT_TASK_NAME +fi + +# Execution of the command itself +if [ -z "$CP_SYSTEM_JOBS_RESULTS" ]; then + pipe_log_warn "CP_SYSTEM_JOBS_RESULTS wasn't provided, running command with local output only." $INIT_TASK_NAME + pipe_exec "$SCRIPTS_DIR/$CP_SYSTEM_SCRIPTS_LOCATION/$CP_SYSTEM_JOB $CP_SYSTEM_JOB_PARAMS" $MAIN_TASK_NAME +else + pipe_exec "$SCRIPTS_DIR/$CP_SYSTEM_SCRIPTS_LOCATION/$CP_SYSTEM_JOB $CP_SYSTEM_JOB_PARAMS | tee $ANALYSIS_DIR/${RUN_ID}.$CP_SYSTEM_JOB.result" $MAIN_TASK_NAME +fi diff --git a/workflows/pipe-templates/__SYSTEM/system_jobs/src/system-jobs/help b/workflows/pipe-templates/__SYSTEM/system_jobs/src/system-jobs/help new file mode 100644 index 0000000000..599830d585 --- /dev/null +++ b/workflows/pipe-templates/__SYSTEM/system_jobs/src/system-jobs/help @@ -0,0 +1,7 @@ +#!/bin/bash + +echo " Available commands: " +echo +for f in `ls ${CP_DOLLAR}SCRIPTS_DIR/src/system-jobs/`; do + echo " $f" +done