Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(pipeline executions/orca) : Added code to save multiple pipelines at once to sql database. #3987

Closed
wants to merge 1 commit into from
Closed

feat(pipeline executions/orca) : Added code to save multiple pipelines at once to sql database. #3987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/Front50Service.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ interface Front50Service {
@POST("/pipelines")
Response savePipeline(@Body Map pipeline, @Query("staleCheck") boolean staleCheck)

@POST("/pipelines/bulksave")
Response savePipelineList(@Body List<Map<String, Object>> pipelineList, @Query("staleCheck") boolean staleCheck)

@PUT("/pipelines/{pipelineId}")
Response updatePipeline(@Path("pipelineId") String pipelineId, @Body Map pipeline)

Expand Down
12 changes: 11 additions & 1 deletion ...t50/src/main/groovy/com/netflix/spinnaker/orca/front50/config/Front50Configuration.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository
import com.netflix.spinnaker.orca.retrofit.RetrofitConfiguration
import com.netflix.spinnaker.orca.retrofit.logging.RetrofitSlf4jLog
import groovy.transform.CompileStatic
import okhttp3.OkHttpClient
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.beans.factory.annotation.Value
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression
Expand All @@ -41,6 +42,8 @@ import retrofit.RequestInterceptor
import retrofit.RestAdapter
import retrofit.converter.JacksonConverter

import java.util.concurrent.TimeUnit

import static retrofit.Endpoints.newFixedEndpoint

@Configuration
Expand All @@ -63,6 +66,9 @@ class Front50Configuration {
@Autowired
RequestInterceptor spinnakerRequestInterceptor

@Value('${okhttp.timeout:10}')
Integer okhttpTimeout

@Bean
Endpoint front50Endpoint(
@Value('${front50.base-url}') String front50BaseUrl) {
Expand All @@ -71,10 +77,14 @@ class Front50Configuration {

@Bean
Front50Service front50Service(Endpoint front50Endpoint, ObjectMapper mapper) {
OkHttpClient okHttpClient = clientProvider.getClient(new DefaultServiceEndpoint("front50", front50Endpoint.getUrl())); println(' timeout : ' + okhttpTimeout)
okHttpClient = okHttpClient.newBuilder().readTimeout(okhttpTimeout, TimeUnit.SECONDS)
.writeTimeout(okhttpTimeout, TimeUnit.SECONDS)
.connectTimeout(okhttpTimeout, TimeUnit.SECONDS).build();
new RestAdapter.Builder()
.setRequestInterceptor(spinnakerRequestInterceptor)
.setEndpoint(front50Endpoint)
.setClient(new Ok3Client(clientProvider.getClient(new DefaultServiceEndpoint("front50", front50Endpoint.getUrl()))))
.setClient(new Ok3Client(okHttpClient))
.setLogLevel(retrofitLogLevel)
.setLog(new RetrofitSlf4jLog(Front50Service))
.setConverter(new JacksonConverter(mapper))
Expand Down
76 changes: 47 additions & 29 deletions orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/tasks/SavePipelineTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,43 +57,61 @@ public TaskResult execute(StageExecution stage) {
throw new IllegalArgumentException("pipeline context must be provided");
}

Map<String, Object> pipeline;
Map<String, Object> pipeline = null;
List<Map<String, Object>> pipelineList = new ArrayList<>();
Boolean staleCheck = false;
Boolean isSavingMultiplePipelines = false;
boolean bulksave = false;
if (!(stage.getContext().get("pipeline") instanceof String)) {
pipeline = (Map<String, Object>) stage.getContext().get("pipeline");
} else if (stage.getContext().containsKey("bulksave")
&& (boolean) stage.getContext().get("bulksave")) {
pipelineList = (List) stage.decodeBase64("/pipeline", List.class);
bulksave = true;
} else {
pipeline = (Map<String, Object>) stage.decodeBase64("/pipeline", Map.class);
pipelineList.add(pipeline);
}

if (!pipeline.containsKey("index")) {
Map<String, Object> existingPipeline = fetchExistingPipeline(pipeline);
if (existingPipeline != null) {
pipeline.put("index", existingPipeline.get("index"));
for (Map<String, Object> obj : pipelineList) {
pipeline = obj;
if (!pipeline.containsKey("index")) {
Map<String, Object> existingPipeline = fetchExistingPipeline(pipeline);
if (existingPipeline != null) {
pipeline.put("index", existingPipeline.get("index"));
}
}
String serviceAccount = (String) stage.getContext().get("pipeline.serviceAccount");
if (serviceAccount != null) {
updateServiceAccount(pipeline, serviceAccount);
}
isSavingMultiplePipelines =
(Boolean)
Optional.ofNullable(stage.getContext().get("isSavingMultiplePipelines"))
.orElse(false);
staleCheck =
(Boolean) Optional.ofNullable(stage.getContext().get("staleCheck")).orElse(false);
if (stage.getContext().get("pipeline.id") != null
&& pipeline.get("id") == null
&& !isSavingMultiplePipelines) {
pipeline.put("id", stage.getContext().get("pipeline.id"));

// We need to tell front50 to regenerate cron trigger id's
pipeline.put("regenerateCronTriggerIds", true);
}

Map<String, Object> finalPipeline = pipeline;
Map<String, Object> finalPipeline1 = pipeline;
pipelineModelMutators.stream()
.filter(m -> m.supports(finalPipeline))
.forEach(m -> m.mutate(finalPipeline1));
}
String serviceAccount = (String) stage.getContext().get("pipeline.serviceAccount");
if (serviceAccount != null) {
updateServiceAccount(pipeline, serviceAccount);
}
final Boolean isSavingMultiplePipelines =
(Boolean)
Optional.ofNullable(stage.getContext().get("isSavingMultiplePipelines")).orElse(false);
final Boolean staleCheck =
(Boolean) Optional.ofNullable(stage.getContext().get("staleCheck")).orElse(false);
if (stage.getContext().get("pipeline.id") != null
&& pipeline.get("id") == null
&& !isSavingMultiplePipelines) {
pipeline.put("id", stage.getContext().get("pipeline.id"));

// We need to tell front50 to regenerate cron trigger id's
pipeline.put("regenerateCronTriggerIds", true);
Response response = null;
if (bulksave) {
response = front50Service.savePipelineList(pipelineList, false);
} else {
response = front50Service.savePipeline(pipeline, staleCheck);
}

pipelineModelMutators.stream()
.filter(m -> m.supports(pipeline))
.forEach(m -> m.mutate(pipeline));

Response response = front50Service.savePipeline(pipeline, staleCheck);

Map<String, Object> outputs = new HashMap<>();
outputs.put("notification.type", "savepipeline");
outputs.put("application", pipeline.get("application"));
Expand All @@ -102,7 +120,7 @@ public TaskResult execute(StageExecution stage) {
try {
Map<String, Object> savedPipeline =
(Map<String, Object>) objectMapper.readValue(response.getBody().in(), Map.class);
outputs.put("pipeline.id", savedPipeline.get("id"));
outputs.put("bulksave", savedPipeline);
} catch (Exception e) {
log.error("Unable to deserialize saved pipeline, reason: ", e.getMessage());

Expand Down
3 changes: 3 additions & 0 deletions orca-web/config/orca.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ tasks:
executionWindow:
timezone: ${global.spinnaker.timezone:America/Los_Angeles}

okhttp:
timeout: 10

logging:
config: classpath:logback-defaults.xml

Expand Down