Merge branch 'develop' into f_aws_native_infra

# Conflicts:
#	deploy/contents/install/app/install-utils.sh
#	deploy/contents/k8s/cp-api-srv/cp-api-srv-dpl.yaml
#	deploy/contents/k8s/cp-search/cp-search-kibana-dpl.yaml

.appveyor.yml

@@ -15,10 +15,16 @@ matrix:
 environment:
   matrix:
     - job_name: CLI_MacOS
-      appveyor_build_worker_image: Previous macos
+      appveyor_build_worker_image: macos-catalina
+    - job_name: CLI_MacOS_Arm
+      appveyor_build_worker_image: macos-catalina
+      appveyor_build_worker_cloud: ip-172-31-13-99.eu-central-1.compute.internal
+    - job_name: Publish_Docs
+      appveyor_build_worker_image: Ubuntu1604
     - job_name: Build_All
       job_depends_on: CLI_MacOS
       appveyor_build_worker_image: Ubuntu1604
+
 for:
 
   -
@@ -29,6 +35,15 @@ for:
     build_script:
     - |-
       bash deploy/appveyor/appveyor_build_macos.sh
+
+  -
+    matrix:
+      only:
+        - job_name: CLI_MacOS_Arm
+
+    build_script:
+    - |-
+      bash deploy/appveyor/appveyor_build_macos_arm.sh
   
   -
     matrix:
@@ -40,4 +55,14 @@ for:
       sudo ln -s "$NVM_DIR/versions/node/$(nvm version)/bin/node" "/usr/local/bin/node"
       sudo ln -s "$NVM_DIR/versions/node/$(nvm version)/bin/npm" "/usr/local/bin/npm"
       sudo -E bash deploy/appveyor/appveyor_pack_dist.sh
+
+  -
+    matrix:
+      only:
+        - job_name: Publish_Docs
+
+    build_script:
+    - |-
+      sudo pip install mkdocs
+      sudo -E bash deploy/appveyor/appveyor_publish_docs.sh
 test: off

api/build.gradle

@@ -40,6 +40,11 @@ task copyFsAutoscalerScripts(type: Copy) {
     into "$rootDir/api/src/main/resources/static/"
 }
 
+task copyEmergencyNodeTerminationScript(type: Copy) {
+    from "$rootDir/scripts/autoscaling/emergency_node_terminator.sh"
+    into "$rootDir/api/src/main/resources/static/"
+}
+
 task copyLocalDtsStartupScripts(type: Copy) {
     from "$rootDir/scripts/dts/DeployDts.ps1", "$rootDir/scripts/dts/deploy_dts.sh"
     into "$rootDir/api/src/main/resources/static/"
@@ -141,7 +146,7 @@ dependencies {
 
     //DB
     compile("org.springframework:spring-jdbc")
-    compile group: "org.postgresql", name: "postgresql", version: "42.1.4"
+    compile group: "org.postgresql", name: "postgresql", version: "42.7.3"
     compile group: "com.mchange", name: "c3p0", version: "0.9.5.2"
     compile group: 'org.hibernate', name: 'hibernate-java8', version: '5.0.12.Final'
 
@@ -155,6 +160,9 @@ dependencies {
     // Core library
     compile project(":core")
 
+    // Event sourcing client module to work with Redis Streams
+    compile project(":cloud-pipeline-common:event-sourcing-java-client")
+
     //Kubernetes
     compile "io.fabric8:kubernetes-client:2.2.7"
     compile "io.fabric8:kubernetes-model:2.1.2"
@@ -199,7 +207,7 @@ dependencies {
     // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-iam
     compile group: "com.amazonaws", name: "aws-java-sdk-iam", version: "1.11.211"
     // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-fsx
-    compile group: 'com.amazonaws', name: 'aws-java-sdk-fsx', version: '1.12.321'
+    compile group: 'com.amazonaws', name: 'aws-java-sdk-fsx', version: '1.12.770'
     compile group: 'com.amazonaws', name: 'aws-java-sdk-route53', version: '1.11.880'
 
 
@@ -258,4 +266,5 @@ dependencies {
 
 // >>>>> processes profiles
 processResources.dependsOn.addAll([copyConfiguration, copyLaunchScripts, copyCommitRunScripts, copyFsAutoscalerScripts,
-                                   copyLocalDtsStartupScripts, generateComponentsVersions])
+                                   copyLocalDtsStartupScripts, generateComponentsVersions,
+                                   copyEmergencyNodeTerminationScript])

api/profiles/dev/application.properties

@@ -17,7 +17,8 @@ spring.lifecycle.timeout-per-shutdown-phase=${CP_API_SRV_SHUTDOWN_TIMEOUT:30}s
 #Security
 api.security.anonymous.urls=${CP_API_SRV_ANONYMOUS_URLS:/restapi/route,/restapi/whoami,/restapi/static-resources/**}
 api.security.impersonation.operations.root.url=${CP_API_SECURITY_IMPERSONATION_ROOT_URL:/restapi/user/impersonation}
-api.security.public.urls=${CP_API_SECURITY_PUBLIC_URLS:/init.sh,/launch.sh,/launch.py,/PipelineCLI.tar.gz,/pipe-common.tar.gz,/commit-run-scripts/**,/pipe,/fsbrowser.tar.gz,/gpustat.tar.gz,/pipe.zip,/pipe.tar.gz,/pipe-el6,/pipe-el6.tar.gz,/pipe-osx,/pipe-osx.tar.gz,/cloud-data-linux.tar.gz,/cloud-data-win64.zip,/fsautoscale.sh,/data-transfer-service.jar,/data-transfer-service-windows.zip,/data-transfer-service-linux.zip,/DeployDts.ps1,/deploy_dts.sh}
+api.security.public.urls=${CP_API_SECURITY_PUBLIC_URLS:/init.sh,/launch.sh,/launch.py,/PipelineCLI.tar.gz,/pipe-common.tar.gz,/commit-run-scripts/**,/pipe,/fsbrowser.tar.gz,/gpustat.tar.gz,/pipe.zip,/pipe.tar.gz,/pipe-el6,/pipe-el6.tar.gz,/pipe-osx,/pipe-osx.tar.gz,/pipe-osx-arm,/pipe-osx-arm.tar.gz,/cloud-data-linux.tar.gz,/cloud-data-win64.zip,/fsautoscale.sh,/data-transfer-service.jar,/data-transfer-service-windows.zip,/data-transfer-service-linux.zip,/DeployDts.ps1,/deploy_dts.sh}
+api.security.swagger.access.roles=${CP_API_SECURITY_SWAGGER_ACCESS_ROLES:ROLE_ADMIN,ROLE_USER}
 
 #db configuration
 database.url=jdbc:postgresql://localhost:5432/pipeline
@@ -77,6 +78,9 @@ commit.run.scripts.root.url=
 commit.run.script.starter.url=
 docker.registry.login.script=
 container.layers.script.url=
+container.size.script.url=
+limit.run.bandwidth.script.url=
+
 
 #pause/resume run scripts
 pause.run.script.url=
@@ -96,6 +100,7 @@ scheduled.quartz.db.driverDelegateClass=${CP_API_SCHEDULING_QUARTZ_DATABASE_DRIV
 scheduled.notifications.cleanup.sec=${CP_API_SCHEDULED_NOTIFICATIONS_CLEANUP:86400}
 
 run.as.pool.size=5
+background.api.jobs.pool.size=${CP_API_BACKGROUND_JOBS_POOL_SIZE:10}
 
 #luigi
 kube.namespace=default
@@ -213,6 +218,14 @@ migration.alias.file=${CP_API_MIGRATION_ALIAS_FILE:}
 #Cache
 cache.type=MEMORY
 
+#Event sourcing
+event.sourcing.enabled=${CP_EVENT_SOURCING_ENABLED:false}
+event.sourcing.redis.host=${CP_EVENT_SOURCING_REDIS_INTERNAL_HOST:cp-redis.default.svc.cluster.local}
+event.sourcing.redis.port=${CP_EVENT_SOURCING_REDIS_INTERNAL_PORT:30097}
+event.sourcing.scheduler.threads=${CP_EVENT_SOURCING_SCHEDULER_THREADS:2}
+event.sourcing.redisson.threads=${CP_EVENT_SOURCING_REDISSON_THREADS:2}
+event.sourcing.redisson.netty.threads=${CP_EVENT_SOURCING_REDISSON_NETTY_THREADS:2}
+
 #edge
 edge.internal.host=${CP_EDGE_INTERNAL_HOST:cp-edge.default.svc.cluster.local}
 edge.internal.port=${CP_EDGE_INTERNAL_PORT:31081}

api/profiles/release/application.properties

@@ -13,6 +13,7 @@ server.session.timeout=1800
 #Security
 api.security.anonymous.urls=${CP_API_SRV_ANONYMOUS_URLS:/restapi/route}
 api.security.impersonation.operations.root.url=${CP_API_SECURITY_IMPERSONATION_ROOT_URL:/restapi/user/impersonation}
+api.security.swagger.access.roles=${CP_API_SECURITY_SWAGGER_ACCESS_ROLES:ROLE_ADMIN,ROLE_USER}
 
 #db configuration
 database.url=

api/src/main/java/com/epam/pipeline/acl/cluster/ClusterApiService.java

@@ -28,23 +28,29 @@
 import com.epam.pipeline.entity.cluster.MasterNode;
 import com.epam.pipeline.entity.cluster.NodeDisk;
 import com.epam.pipeline.entity.cluster.NodeInstance;
+import com.epam.pipeline.entity.cluster.PodDescription;
+import com.epam.pipeline.entity.cluster.PodInstance;
 import com.epam.pipeline.entity.cluster.monitoring.MonitoringStats;
+import com.epam.pipeline.entity.cluster.monitoring.gpu.GpuMetricsGranularity;
+import com.epam.pipeline.entity.cluster.monitoring.gpu.GpuMonitoringStats;
 import com.epam.pipeline.entity.pipeline.run.RunInfo;
 import com.epam.pipeline.manager.cluster.EdgeServiceManager;
 import com.epam.pipeline.manager.cluster.InstanceOfferManager;
 import com.epam.pipeline.manager.cluster.MonitoringReportType;
 import com.epam.pipeline.manager.cluster.NodeDiskManager;
 import com.epam.pipeline.manager.cluster.NodesManager;
+import com.epam.pipeline.manager.cluster.PodsManager;
 import com.epam.pipeline.manager.cluster.performancemonitoring.UsageMonitoringManager;
 import com.epam.pipeline.manager.security.acl.AclMask;
 import lombok.RequiredArgsConstructor;
 import org.springframework.security.access.prepost.PostFilter;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Service;
 
+import static com.epam.pipeline.security.acl.AclExpressions.ADMIN_ONLY;
+import static com.epam.pipeline.security.acl.AclExpressions.ADMIN_OR_GENERAL_USER;
 import static com.epam.pipeline.security.acl.AclExpressions.NODE_READ;
 import static com.epam.pipeline.security.acl.AclExpressions.NODE_READ_FILTER;
-import static com.epam.pipeline.security.acl.AclExpressions.NODE_USAGE_READ;
 import static com.epam.pipeline.security.acl.AclExpressions.NODE_STOP;
 
 @Service
@@ -56,6 +62,7 @@ public class ClusterApiService {
     private final UsageMonitoringManager usageMonitoringManager;
     private final InstanceOfferManager instanceOfferManager;
     private final EdgeServiceManager edgeServiceManager;
+    private final PodsManager podsManager;
 
     @PostFilter(NODE_READ_FILTER)
     public List<NodeInstance> getNodes() {
@@ -90,14 +97,23 @@ public NodeInstance terminateNode(final String name) {
         return nodesManager.terminateNode(name);
     }
 
-    @PreAuthorize(NODE_USAGE_READ)
+    @PreAuthorize(ADMIN_OR_GENERAL_USER)
     public List<MonitoringStats> getStatsForNode(final String name,
                                                  final LocalDateTime from,
                                                  final LocalDateTime to) {
         return usageMonitoringManager.getStatsForNode(name, from, to);
     }
 
-    @PreAuthorize(NODE_USAGE_READ)
+    @PreAuthorize(ADMIN_OR_GENERAL_USER)
+    public GpuMonitoringStats getGpuStatsForNode(final String name,
+                                                 final LocalDateTime from,
+                                                 final LocalDateTime to,
+                                                 final List<GpuMetricsGranularity> granularity,
+                                                 final boolean squashCharts) {
+        return usageMonitoringManager.getGpuStatsForNode(name, from, to, granularity, squashCharts);
+    }
+
+    @PreAuthorize(ADMIN_OR_GENERAL_USER)
     public InputStream getUsageStatisticsFile(final String name, final LocalDateTime from, final LocalDateTime to,
                                               final Duration interval, final MonitoringReportType type) {
         return usageMonitoringManager.getStatsForNodeAsInputStream(name, from, to, interval, type);
@@ -128,4 +144,19 @@ public List<NodeDisk> loadNodeDisks(final String name) {
     public String buildEdgeExternalUrl(final String region) {
         return edgeServiceManager.buildEdgeExternalUrl(region);
     }
+
+    @PreAuthorize(ADMIN_ONLY)
+    public List<PodInstance> getCorePods() {
+        return podsManager.getCorePods();
+    }
+
+    @PreAuthorize(ADMIN_ONLY)
+    public PodDescription getPodDescription(final String podId, final boolean detailed) {
+        return podsManager.describePod(podId, detailed);
+    }
+
+    @PreAuthorize(ADMIN_ONLY)
+    public String getContainerLogs(final String podId, final String containerId, final Integer limit) {
+        return podsManager.getContainerLogs(podId, containerId, limit);
+    }
 }

api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java

@@ -20,6 +20,7 @@
 import com.epam.pipeline.common.MessageHelper;
 import com.epam.pipeline.controller.vo.DataStorageVO;
 import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.controller.vo.security.EntityWithPermissionVO;
 import com.epam.pipeline.entity.AbstractSecuredEntity;
 import com.epam.pipeline.entity.SecuredEntityWithAction;
@@ -34,6 +35,7 @@
 import com.epam.pipeline.entity.datastorage.DataStorageItemContent;
 import com.epam.pipeline.entity.datastorage.DataStorageItemType;
 import com.epam.pipeline.entity.datastorage.DataStorageListing;
+import com.epam.pipeline.entity.datastorage.DataStorageListingFilter;
 import com.epam.pipeline.entity.datastorage.DataStorageStreamingContent;
 import com.epam.pipeline.entity.datastorage.DataStorageWithShareMount;
 import com.epam.pipeline.entity.datastorage.PathDescription;
@@ -89,6 +91,11 @@ public List<AbstractDataStorage> getDataStorages() {
         return dataStorageManager.getDataStorages();
     }
 
+    @StorageAclRead
+    public List<AbstractDataStorage> getDataStorages(final EntityFilterVO filter) {
+        return dataStorageManager.getDataStorages(filter);
+    }
+
     @StorageAclReadAndWrite
     public List<AbstractDataStorage> getWritableStorages() {
         return dataStorageManager.getDataStorages();
@@ -135,6 +142,14 @@ public DataStorageListing getDataStorageItems(final Long id, final String path,
         return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived);
     }
 
+    @PreAuthorize(AclExpressions.STORAGE_ID_READ + AclExpressions.AND
+            + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS)
+    public DataStorageListing filterDataStorageItems(final Long id, final String path, final boolean showVersion,
+                                                     final boolean showArchived,
+                                                     final DataStorageListingFilter filter) {
+        return dataStorageManager.filterDataStorageItems(id, path, showVersion, showArchived, filter);
+    }
+
     @PreAuthorize(AclExpressions.STORAGE_ID_OWNER + AclExpressions.AND
             + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS)
     public DataStorageListing getDataStorageItemsOwner(final Long id, final String path,
@@ -143,6 +158,14 @@ public DataStorageListing getDataStorageItemsOwner(final Long id, final String p
         return dataStorageManager.getDataStorageItems(id, path, showVersion, pageSize, marker, showArchived);
     }
 
+    @PreAuthorize(AclExpressions.STORAGE_ID_OWNER + AclExpressions.AND
+            + AclExpressions.STORAGE_SHOW_ARCHIVED_PERMISSIONS)
+    public DataStorageListing filterDataStorageItemsOwner(final Long id, final String path, final boolean showVersion,
+                                                          final boolean showArchived,
+                                                          final DataStorageListingFilter filter) {
+        return dataStorageManager.filterDataStorageItems(id, path, showVersion, showArchived, filter);
+    }
+
     @PreAuthorize(AclExpressions.STORAGE_ID_WRITE)
     public List<AbstractDataStorageItem> updateDataStorageItems(final Long id,
             List<UpdateDataStorageItemVO> list) throws DataStorageException {

api/src/main/java/com/epam/pipeline/acl/datastorage/lustre/LustreFSApiService.java

@@ -30,8 +30,9 @@ public class LustreFSApiService {
 
     @PreAuthorize(AclExpressions.RUN_ID_EXECUTE)
     public LustreFS getOrCreateLustreFS(final Long runId, final Integer size,
-                                        final String type, final Integer throughput) {
-        return lustreFSManager.getOrCreateLustreFS(runId, size, type, throughput);
+                                        final String type, final Integer throughput,
+                                        final Integer iops) {
+        return lustreFSManager.getOrCreateLustreFS(runId, size, type, throughput, iops);
     }
 
     @PreAuthorize(AclExpressions.RUN_ID_EXECUTE)

api/src/main/java/com/epam/pipeline/acl/docker/ToolApiService.java

@@ -20,6 +20,7 @@
 import com.epam.pipeline.entity.docker.ImageDescription;
 import com.epam.pipeline.entity.docker.ImageHistoryLayer;
 import com.epam.pipeline.entity.docker.ToolDescription;
+import com.epam.pipeline.entity.docker.ToolImageDockerfile;
 import com.epam.pipeline.entity.docker.ToolVersion;
 import com.epam.pipeline.entity.docker.ToolVersionAttributes;
 import com.epam.pipeline.entity.pipeline.Tool;
@@ -114,6 +115,11 @@ public List<ImageHistoryLayer> getImageHistory(final Long id, final String tag)
         return toolManager.loadToolHistory(id, tag);
     }
 
+    @PreAuthorize(AclExpressions.TOOL_READ)
+    public ToolImageDockerfile loadDockerFile(final Long id, final String tag, final String from) {
+        return toolManager.loadDockerFile(id, tag, from);
+    }
+
     @PreAuthorize(AclExpressions.TOOL_READ)
     public String getImageDefaultCommand(final Long id, final String tag) {
         return toolManager.loadToolDefaultCommand(id, tag);

api/src/main/java/com/epam/pipeline/acl/pipeline/PipelineApiService.java

@@ -26,6 +26,7 @@
 import com.epam.pipeline.controller.vo.RegisterPipelineVersionVO;
 import com.epam.pipeline.controller.vo.TaskGraphVO;
 import com.epam.pipeline.controller.vo.UploadFileMetadata;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.entity.cluster.InstancePrice;
 import com.epam.pipeline.entity.git.GitCommitEntry;
 import com.epam.pipeline.entity.git.GitCommitsFilter;
@@ -131,6 +132,12 @@ public List<Pipeline> loadAllPipelines(boolean loadVersions) {
         return pipelineManager.loadAllPipelines(loadVersions);
     }
 
+    @PostFilter("hasRole('ADMIN') OR hasPermission(filterObject, 'READ')")
+    @AclMaskList
+    public List<Pipeline> filterPipelines(final boolean loadVersions, final EntityFilterVO filter) {
+        return pipelineManager.loadAllPipelines(loadVersions, filter);
+    }
+
     @PreAuthorize(ADMIN_ONLY)
     public PipelinesWithPermissionsVO loadAllPipelinesWithPermissions(Integer pageNum, Integer pageSize) {
         return permissionManager.loadAllPipelinesWithPermissions(pageNum, pageSize);