Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set sensitive property when launch runPod #3168

Open
wants to merge 15 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,11 @@ public final class MessageConstants {

// Launch limits
public static final String ERROR_RUN_LAUNCH_USER_LIMIT_EXCEEDED = "error.run.launch.user.limit.exceeded";
public static final String ERROR_RUN_ASSIGN_POLICY_MALFORMED = "error.run.assign.policy.malformed";
public static final String ERROR_RUN_ASSIGN_POLICY_FORBIDDEN = "error.run.assign.policy.forbidden";
public static final String ERROR_RUN_WITH_SERVICE_ACCOUNT_FORBIDDEN = "error.run.with.service.account.forbidden";



// Ngs preprocessing
public static final String ERROR_NGS_PREPROCESSING_FOLDER_ID_NOT_PROVIDED =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public Result<Boolean> deleteIssue(@PathVariable(value = ISSUE_ID) final Long is
return Result.success(gitlabIssueApiService.deleteIssue(issueId));
}

@GetMapping
@PostMapping(value = "/filter")
@ApiOperation(
value = "Gets all users issues.",
notes = "Gets all users issues.",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@
import lombok.Getter;
import lombok.RequiredArgsConstructor;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

@Getter
@RequiredArgsConstructor
public enum BillingGrouping {
Expand All @@ -36,4 +41,11 @@ public enum BillingGrouping {
private final boolean runUsageDetailsRequired;
private final boolean storageUsageDetailsRequired;

public static Set<BillingGrouping> getStorageGrouping() {
return Collections.singleton(STORAGE);
}

public static Set<BillingGrouping> getRunGrouping() {
return new HashSet<>(Arrays.asList(RUN_COMPUTE_TYPE, PIPELINE, TOOL, USER, BILLING_CENTER));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,36 @@

package com.epam.pipeline.entity.billing;

import com.epam.pipeline.manager.billing.BillingUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;

import java.util.Set;

@Getter
@RequiredArgsConstructor
public enum BillingGroupingOrderAggregate {

DEFAULT(null, "cost", "cost"),
DEFAULT(null, BillingUtils.COST_FIELD, BillingUtils.COST_FIELD, null),

STORAGE(BillingGrouping.STORAGE, "cost", "usage_bytes"),
STANDARD(BillingGrouping.STORAGE,
"standard_total_cost", "standard_total_usage_bytes"),
GLACIER(BillingGrouping.STORAGE,
"glacier_total_cost", "glacier_total_usage_bytes"),
GLACIER_IR(BillingGrouping.STORAGE,
"glacier_ir_total_cost", "glacier_ir_total_usage_bytes"),
DEEP_ARCHIVE(BillingGrouping.STORAGE,
"deep_archive_total_cost", "deep_archive_total_usage_bytes");
STORAGE(BillingGrouping.getStorageGrouping(), BillingUtils.COST_FIELD, BillingUtils.STORAGE_USAGE_FIELD, null),
STANDARD(BillingGrouping.getStorageGrouping(),
"standard_total_cost", "standard_total_usage_bytes", null),
GLACIER(BillingGrouping.getStorageGrouping(),
"glacier_total_cost", "glacier_total_usage_bytes", null),
GLACIER_IR(BillingGrouping.getStorageGrouping(),
"glacier_ir_total_cost", "glacier_ir_total_usage_bytes", null),
DEEP_ARCHIVE(BillingGrouping.getStorageGrouping(),
"deep_archive_total_cost", "deep_archive_total_usage_bytes", null),
RUN(BillingGrouping.getRunGrouping(), BillingUtils.COST_FIELD,
BillingUtils.RUN_USAGE_FIELD, BillingUtils.RUN_ID_FIELD),
RUN_COMPUTE(BillingGrouping.getRunGrouping(), BillingUtils.COMPUTE_COST_FIELD,
BillingUtils.RUN_USAGE_FIELD, BillingUtils.RUN_ID_FIELD),
RUN_DISK(BillingGrouping.getRunGrouping(), BillingUtils.DISK_COST_FIELD,
BillingUtils.RUN_USAGE_FIELD, BillingUtils.RUN_ID_FIELD);

private final BillingGrouping group;
private final Set<BillingGrouping> groups;
private final String costField;
private final String usageField;
private final String countField;
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,12 @@

package com.epam.pipeline.entity.billing;

import com.epam.pipeline.manager.billing.BillingUtils;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;

@Getter
@RequiredArgsConstructor
Expand All @@ -32,7 +35,7 @@ public class BillingGroupingSortOrder {
);

public enum BillingGroupingSortMetric {
COST, USAGE
COST, USAGE, USAGE_RUNS, COUNT_RUNS
}

private final BillingGroupingSortMetric metric;
Expand All @@ -44,11 +47,29 @@ public String getAggregateToOrderBy() {
case COST:
return aggregate.getCostField();
case USAGE:
case USAGE_RUNS:
return aggregate.getUsageField();
case COUNT_RUNS:
return aggregate.getCountField();
default:
return BillingGroupingOrderAggregate.DEFAULT.getCostField();
}

}

public AggregationBuilder getAggregation() {
final String aggregateField = getAggregateToOrderBy();
switch (metric) {
case USAGE:
return AggregationBuilders.avg(aggregateField + BillingUtils.SORT_AGG_POSTFIX)
.field(aggregateField);
case COUNT_RUNS:
return AggregationBuilders.count(aggregateField + BillingUtils.SORT_AGG_POSTFIX)
.field(aggregateField);
default:
return AggregationBuilders.sum(aggregateField + BillingUtils.SORT_AGG_POSTFIX)
.field(aggregateField);
}
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ public final class BillingUtils {
public static final String DISCOUNT_SCRIPT_TEMPLATE = "_value + _value * (%s)";
public static final String RESOURCE_TYPE = "resource_type";
public static final String COMPUTE_GROUP = "COMPUTE";
public static final String SORT_AGG_POSTFIX = "_sort_order";

private BillingUtils() {
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,59 +19,33 @@
import com.epam.pipeline.entity.billing.BillingGrouping;
import com.epam.pipeline.entity.billing.BillingGroupingOrderAggregate;
import com.epam.pipeline.entity.billing.BillingGroupingSortOrder;
import com.epam.pipeline.manager.billing.BillingUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.springframework.util.Assert;

public final class BillingOrderApplier {

public static final String SORT_AGG_POSTFIX = "_sort_order";

private BillingOrderApplier() {}

public static BoolQueryBuilder applyOrder(final BillingGrouping grouping,
final BillingGroupingSortOrder order,
final BoolQueryBuilder query,
final TermsAggregationBuilder terms) {
final BillingGroupingOrderAggregate orderAggregate = order.getAggregate();
Assert.isTrue(orderAggregate.getGroup() == null || grouping.equals(orderAggregate.getGroup()),
Assert.isTrue(orderAggregate.getGroups() == null || orderAggregate.getGroups().contains(grouping),
String.format("Grouping: %s and Grouping Order: %s, don't match.",
grouping.name(), orderAggregate.name()));

// Apply additional filter to query to filter out docs that don't have value to sort by
query.filter(QueryBuilders.boolQuery().must(QueryBuilders.existsQuery(order.getAggregateToOrderBy())));

// Depends on Grouping type apply specific logic for ordering results
if (grouping == BillingGrouping.STORAGE) {
storageGroupingOrder(order, terms);
} else {
defaultOrder(order, terms);
}
return query;
}
terms.subAggregation(order.getAggregation());
terms.order(BucketOrder.aggregation(order.getAggregateToOrderBy() + BillingUtils.SORT_AGG_POSTFIX,
order.isDesc()));

private static void storageGroupingOrder(final BillingGroupingSortOrder order,
final TermsAggregationBuilder terms) {
final String aggToOrderBy = order.getAggregateToOrderBy();
if (order.getMetric() == BillingGroupingSortOrder.BillingGroupingSortMetric.COST) {
terms.subAggregation(AggregationBuilders.sum(aggToOrderBy + SORT_AGG_POSTFIX).field(aggToOrderBy));
} else {
terms.subAggregation(AggregationBuilders.avg(aggToOrderBy + SORT_AGG_POSTFIX).field(aggToOrderBy));
}
terms.order(BucketOrder.aggregation(aggToOrderBy + SORT_AGG_POSTFIX, order.isDesc()));
}

private static void defaultOrder(final BillingGroupingSortOrder order, final TermsAggregationBuilder terms) {
terms.subAggregation(
AggregationBuilders.sum(order.getAggregateToOrderBy() + SORT_AGG_POSTFIX)
.field(order.getAggregateToOrderBy())
);
terms.order(
BucketOrder.aggregation(order.getAggregateToOrderBy() + SORT_AGG_POSTFIX, order.isDesc())
);
return query;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.epam.pipeline.entity.pipeline.RunLog;
import com.epam.pipeline.entity.pipeline.TaskStatus;
import com.epam.pipeline.entity.pipeline.ToolGroup;
import com.epam.pipeline.entity.pipeline.run.RunAssignPolicy;
import com.epam.pipeline.entity.region.AbstractCloudRegion;
import com.epam.pipeline.entity.utils.DateUtils;
import com.epam.pipeline.exception.CmdExecutionException;
Expand Down Expand Up @@ -331,8 +332,7 @@ private void launchPodIfRequired(final PipelineRun run, final List<String> endpo
try {
if (Objects.isNull(kubernetesManager.findPodById(run.getPodId()))) {
final PipelineConfiguration configuration = getResumeConfiguration(run);
launcher.launch(run, configuration, endpoints, run.getId().toString(),
true, run.getPodId(), null, ImagePullPolicy.NEVER);
launcher.launch(run, configuration, endpoints, true, run.getPodId(), null, ImagePullPolicy.NEVER);
}
} finally {
resumeLock.unlock();
Expand All @@ -343,6 +343,14 @@ private PipelineConfiguration getResumeConfiguration(final PipelineRun run) {
final PipelineConfiguration configuration = configurationManager.getConfigurationFromRun(run);
final Map<String, String> envs = getResumeRunEnvVars(configuration);
configuration.setEnvironmentParams(envs);
configuration.setPodAssignPolicy(
RunAssignPolicy.builder()
.selector(
RunAssignPolicy.PodAssignSelector.builder()
.label(KubernetesConstants.RUN_ID_LABEL)
.value(run.getId().toString()).build())
.build()
);
return configuration;
}

Expand Down
Loading