Skip to content

Commit

Permalink
fix:[CUS-161] Delete Unused VM disk.
Browse files Browse the repository at this point in the history
  • Loading branch information
kushZemoso authored and santhosh-challa committed Aug 21, 2023
1 parent 7657fd8 commit 76ab7b5
Show file tree
Hide file tree
Showing 10 changed files with 197 additions and 18 deletions.
2 changes: 2 additions & 0 deletions installer/resources/pacbot_app/files/DB.sql
Original file line number Diff line number Diff line change
Expand Up @@ -1247,6 +1247,7 @@ INSERT IGNORE INTO `cf_Target` (`targetName`,`displayName`,`targetDesc`,`categor

INSERT IGNORE INTO cf_Target (`targetName`,`targetDesc`,`displayName`,`category`,`dataSourceName`,`targetConfig`,`status`,`userId`,`endpoint`,`createdDate`,`modifiedDate`,`domain`) VALUES ('cloudfunction','GCP Cloud Functions','GCP cloud functions','Security','gcp','{"key":"id","id":"id"}','enabled','admin',concat(@eshost,':',@esport,'/gcp_cloudfunction'),'2023-01-10','2023-01-10','Infra & Platforms');
INSERT IGNORE INTO cf_Target (`targetName`,`targetDesc`,`displayName`,`category`,`dataSourceName`,`targetConfig`,`status`,`userId`,`endpoint`,`createdDate`,`modifiedDate`,`domain`) VALUES ('cloudfunctiongen1','GCP Cloud Functions Generation 1','GCP cloud functions Generation 1','Security','gcp','{"key":"id","id":"id"}','enabled','admin',concat(@eshost,':',@esport,'/gcp_cloudfunctiongen1'),'2023-01-10','2023-01-10','Infra & Platforms');
INSERT IGNORE INTO `cf_Target` (`targetName`,`displayName`, `targetDesc`, `category`, `dataSourceName`, `targetConfig`, `status`, `userId`, `endpoint`, `createdDate`, `modifiedDate`, `domain`) VALUES('gcpdisks','Managed Disks (Gcp)','GCP Disks','security','gcp','{\"key\":\"id\",\"id\":\"id\"}','enabled','admin@pacbot.org',concat(@eshost,':',@esport,'/gcp_gcpdisks/gcpdisks'),'2022-12-5','2022-12-5','Infra & Platforms');

INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeName,attributeValue) VALUES ('11501','201','ec2','all','all');
INSERT IGNORE INTO cf_AssetGroupTargetDetails (id_,groupId,targetType,attributeName,attributeValue) VALUES ('11502','201','s3','all','all');
Expand Down Expand Up @@ -1375,6 +1376,7 @@ INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`,
INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('de364119-0f2b-4f63-8d61-81fa4d1d33fb','e0008397-f74e-4deb-9066-10bdf11202ae','iamusers','all','all');
INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('25e615a5-e7d3-444e-95a3-2dedaef0890e','e0008397-f74e-4deb-9066-10bdf11202ae','gcp_apikeys','all','all');
INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('9b942f42-4bd0-4911-8fd3-a1661f0cbc97','e0008397-f74e-4deb-9066-10bdf11202ae','gcp_loadbalancers','all','all');
INSERT IGNORE INTO `cf_AssetGroupTargetDetails` (`id_`, `groupId`, `targetType`, `attributeName`, `attributeValue`) VALUES('48df4f33-62c9-42c6-8fb0-0bc69bad3e37','e0008397-f74e-4deb-9066-10bdf11202ae','gcpdisks','all','all');



Expand Down
4 changes: 4 additions & 0 deletions installer/resources/pacbot_app/files/DB_Policy.sql
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,10 @@ DELETE IGNORE FROM cf_PolicyTable where policyUUID='tenable_aws_vm_scanned_rul
DELETE IGNORE FROM cf_PolicyTable where policyUUID='remove_unused_scale_set';
INSERT IGNORE INTO cf_PolicyTable (policyId, policyUUID, policyName, policyDisplayName, policyDesc, resolution, resolutionUrl, targetType, assetGroup, alexaKeyword, policyParams, policyFrequency, policyExecutable, policyRestUrl, policyType, policyArn, severity, category, autoFixAvailable, autoFixEnabled, allowList, waitingTime, maxEmailNotification, templateName, templateColumns, fixType, warningMailSubject, fixMailSubject, warningMessage, fixMessage, violationMessage, elapsedTime, userId, createdDate, modifiedDate, status) VALUES ('remove_unused_scale_set','remove_unused_scale_set',' Delete Unused Scale Set',' Delete Unused Scale Set','Identify any empty virtual machine scale sets available within your Microsoft Azure cloud account and delete them in order to eliminate unnecessary costs and meet compliance requirements when it comes to unused resources.','Every empty virtual machine scale set should be removed for cost optimization and better management of your cloud resources.','','virtualmachinescaleset','azure','remove_unused_scale_set','{"params":[{"encrypt":false,"value":"check-for-unused-Virtual-machine-scale-set","key":"policyKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags","isMandatory":true,"description":"Assets should have these mandatory tags","defaultVal":"Application,Environment,Stack,Role","displayName":"Mandatory tags"},{"encrypt":false,"value":"low","key":"severity"},{"encrypt":false,"value":"security","key":"policyCategory"},{"encrypt":false,"value":"","key":"policyOwner"}],"environmentVariables":[],"policyId":"remove_unused_scale_set","autofix":false,"alexaKeyword":"remove_unused_scale_set","policyRestUrl":"","targetType":"virtualmachinescaleset","pac_ds":"azure","assetGroup":"azure","policyUUID":"remove_unused_scale_set","policyType":"ManagePolicy"}','0 0/6 * * ? *','','','ManagePolicy','arn:aws:events:us-east-1:***REMOVED***:rule/remove_unused_scale_set','low','security','false','false',NULL,24,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,24,'ASGC','2023-07-19','2023-07-19','ENABLED');

DELETE IGNORE FROM cf_PolicyTable where policyUUID='delete_unused_vm_disk';
INSERT IGNORE INTO cf_PolicyTable (policyId, policyUUID, policyName, policyDisplayName, policyDesc, resolution, resolutionUrl, targetType, assetGroup, alexaKeyword, policyParams, policyFrequency, policyExecutable, policyRestUrl, policyType, policyArn, severity, category, autoFixAvailable, autoFixEnabled, allowList, waitingTime, maxEmailNotification, templateName, templateColumns, fixType, warningMailSubject, fixMailSubject, warningMessage, fixMessage, violationMessage, elapsedTime, userId, createdDate, modifiedDate, status) VALUES ('delete_unused_vm_disk','delete_unused_vm_disk',' Delete Unused VM Disk','Delete Unused VM Disk','Identify any unattached (unused) Gcp virtual machine disk volumes available within yourcloud account and delete them in order to lower the cost of your monthly bill and reduce the risk of sensitive data leakage.','Every unused virtual machine disk should be removed for cost optimization and better management of your cloud resources.','','gcpdisks','gcp','delete_unused_vm_disk','{"params":[{"encrypt":false,"value":"delete-unused-vm-disk","key":"policyKey"},{"encrypt":false,"value":",","key":"splitterChar"},{"encrypt":false,"value":"Application,Environment,Stack,Role","key":"mandatoryTags","isMandatory":true,"description":"Assets should have these mandatory tags","defaultVal":"Application,Environment,Stack,Role","displayName":"Mandatory tags"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"cost","key":"policyCategory"},{"encrypt":false,"value":"","key":"policyOwner"}],"environmentVariables":[],"policyId":"delete_unused_vm_disk","autofix":false,"alexaKeyword":"delete_unused_vm_disk","policyRestUrl":"","targetType":"gcpdisks","pac_ds":"gcp","assetGroup":"gcp","policyUUID":"delete_unused_vm_disk","policyType":"ManagePolicy"}','0 0/6 * * ? *','','','ManagePolicy','arn:aws:events:us-east-1:***REMOVED***:rule/delete_unused_vm_disk','high','cost','false','false',NULL,24,1,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,24,'ASGC','2023-07-19','2023-07-19','ENABLED');
update cf_PolicyTable set resolutionUrl='https://paladincloud.io/docs/gcp-policy/#articleTOC_104',policyDesc='Deleting unused VM disks in GCP is essential for cost savings, resource management, security, and performance optimization. It streamlines your cloud environment, reduces expenses, and ensures compliance with data protection regulations.' where policyId='delete_unused_vm_disk';

INSERT IGNORE INTO cf_PolicyTable (policyId,policyUUID,policyName,policyDisplayName,policyDesc,resolution,resolutionUrl,targetType,assetGroup,alexaKeyword,policyParams,policyFrequency,policyExecutable,policyRestUrl,policyType,policyArn,status,userId,createdDate,modifiedDate,severity,category,autoFixAvailable,autoFixEnabled,allowList,waitingTime,maxEmailNotification,templateName,templateColumns,fixType,warningMailSubject,fixMailSubject,warningMessage,fixMessage,violationMessage,elapsedTime)VALUES('AWSVMScannedByTenable','tenable_aws_vm_scanned_rule','Scanning Amazon VMs with Tenable Security','Scan Amazon VM with Tenable Security','Tenable Security\'s container security platform can scan VMs in a client\'s Amazon account to identify any VMs not scanned for vulnerabilities. There are two possible reasons for a resource (VM) not being scanned: it has no vulnerabilities, or its metadata has not been collected by cloud discovery. The primary objective of this process is to ensure that all VMs in the account are scanned for vulnerabilities, thereby mitigating potential security risks.','create/register in tenable Sass platform and add AWS connector for scan and get it scanned regularly','https://github.com/PaladinCloud/CE/wiki/AWS-Policy#Scan-Amazon-EC2-Image-with-Tenable-Security','ec2','aws','AWSVMScannedByTenable','{"assetGroup":"aws","policyId":"AWSVMScannedByTenable","policyRestUrl":"","environmentVariables":[],"policyUUID":"tenable_aws_vm_scanned_rule","policyType":"ManagePolicy","pac_ds":"aws","targetType":"ec2","params":[{"defaultVal":"30","encrypt":false,"isEdit":true,"displayName":"Target","description":"Target in days","value":"30","key":"target","isMandatory":true},{"isValueNew":true,"encrypt":false,"value":"/tenable-vm-vulnerability/_search","key":"esTenableVMUrl"},{"isValueNew":true,"defaultVal":"7","encrypt":false,"isEdit":true,"displayName":"Discovered days range","description":"Discovered days in number","value":"7","key":"discoveredDaysRange","isMandatory":true},{"isValueNew":true,"encrypt":false,"value":"check-for-vms-scanned-by-tenable","key":"policyKey"},{"encrypt":false,"value":"high","key":"severity"},{"encrypt":false,"value":"security","key":"policyCategory"}],"autofix":false,"alexaKeyword":"AWSVMScannedByTenable"}','0 0 ? * MON *','','','ManagePolicy','arn:aws:events:us-east-1:***REMOVED***:rule/tenable_aws_vm_scanned_rule','ENABLED','ASGC',now(),now(),'high','security',false,false,null,24,1,null,null,null,null,null,null,null,null,24);
DELETE IGNORE FROM cf_PolicyTable where policyUUID='aws_vm_high_vulnerabilities_tenable_scanned_rule';
DELETE IGNORE FROM cf_PolicyTable where policyUUID='tenable_aws_vm_high_vulnerabilities_scanned_rule';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ public class GCPCredentialsProvider {

private ApiKeysClient apiKeysClient;

private DisksClient disksClient;

private Map<String, GoogleCredentials> credentialCache = new HashMap<>();

// If you don't specify credentials when constructing the client, the client
Expand Down Expand Up @@ -351,6 +353,14 @@ public SslPoliciesClient getSslPoliciesClient(String projectId) throws IOExcep
return sslPoliciesClient;
}

public DisksClient getDiskClient(String projectId) throws IOException{
if(disksClient==null){
DisksSettings disksSettings=DisksSettings.newBuilder().setCredentialsProvider(FixedCredentialsProvider.create(this.getCredentials(projectId))).build();
disksClient=DisksClient.create(disksSettings);
}
return disksClient;
}



/*public CloudFunctionsServiceClient getFunctionClientGen1(String projectId) throws IOException {
Expand Down Expand Up @@ -390,5 +400,6 @@ public void nullifyAllGcpClients(){
this.backendService=null;
this.targetHttpsProxiesClient=null;
this.sslPoliciesClient=null;
this.disksClient=null;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package com.tmobile.pacbot.gcp.inventory.collector;


import com.google.cloud.compute.v1.Disk;
import com.google.cloud.compute.v1.DisksClient;
import com.google.cloud.compute.v1.ListDisksRequest;
import com.tmobile.pacbot.gcp.inventory.auth.GCPCredentialsProvider;
import com.tmobile.pacbot.gcp.inventory.vo.DiskVH;
import com.tmobile.pacbot.gcp.inventory.vo.ProjectVH;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.cloud.compute.v1.DisksClient.ListPagedResponse;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class DiskInventoryCollector {
@Autowired
GCPCredentialsProvider gcpCredentialsProvider;

private static final Logger logger = LoggerFactory.getLogger(DiskInventoryCollector.class);

public List<DiskVH> fetchDiskInventory(ProjectVH project) throws IOException {
List<DiskVH> diskList = new ArrayList<>();
logger.debug("Project id:{}",project.getProjectNumber());

DisksClient disksClient=gcpCredentialsProvider.getDiskClient(project.getProjectId());
ListDisksRequest request = ListDisksRequest.newBuilder()
.setProject(project.getProjectId())
.build();
ListPagedResponse diskResponse = disksClient.list(request);
logger.info("Disk entry {}", diskResponse);
for (Disk disk : diskResponse.iterateAll()) {
logger.info("Disk iterator {}", disk);

DiskVH diskVH=new DiskVH();
diskVH.setName(disk.getName());
diskVH.setKind(disk.getKind());
diskVH.setSizeGb(disk.getSizeGb());
diskVH.setZone(disk.getZone());
diskVH.setStatus(disk.getStatus());
diskVH.setType(disk.getType());
diskVH.setId(String.valueOf(disk.getId()));
diskVH.setProjectName(project.getProjectName());
diskVH.setProjectId(project.getProjectId());
diskVH.setLicenses(disk.getLicensesList());
diskVH.setUsers(disk.getUsersList());
diskVH.setLicenseCodes(disk.getLicenseCodesList());
diskList.add(diskVH);

logger.info("Disk exit {}", diskVH);
}

logger.info("Disk list {}", diskList);

return diskList;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ public class AssetFileGenerator {
@Autowired
APIKeysInventoryCollector apiKeysInventoryCollector;

@Autowired
DiskInventoryCollector diskInventoryCollector;

@Autowired
RDSDBManager rdsdbManager;

Expand Down Expand Up @@ -325,6 +328,17 @@ public void generateFiles(List<ProjectVH> projects, String filePath) {
Util.errorCount.getAndIncrement();
}
});
executor.execute(() -> {
if (!(isTypeInScope("gcpdisks"))) {
return;
}
try {
FileManager.generateDisksFiles(diskInventoryCollector.fetchDiskInventory(project));
} catch (Exception e) {
e.printStackTrace();
Util.errorCount.getAndIncrement();
}
});
/*executor.execute(() -> {
if (!(isTypeInScope("cloudfunctiongen1"))) {
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public static void initialise(String folderName) throws IOException {
FileGenerator.writeToFile("gcp-iamusers.data", "[", false);
FileGenerator.writeToFile("gcp-gcploadbalancer.data", "[", false);
FileGenerator.writeToFile("gcp-apikeys.data", "[", false);
FileGenerator.writeToFile("gcp-gcpdisks.data", "[", false);
FileGenerator.writeToFile(DataFileNamesConstants.CLOUD_FUNCTION, "[", false);
FileGenerator.writeToFile(DataFileNamesConstants.CLOUD_FUNCTION_GEN1, "[", false);
}
Expand All @@ -93,6 +94,7 @@ public static void finalise() throws IOException {
FileGenerator.writeToFile("gcp-iamusers.data", "]", true);
FileGenerator.writeToFile("gcp-gcploadbalancer.data", "]", true);
FileGenerator.writeToFile("gcp-apikeys.data", "]", true);
FileGenerator.writeToFile("gcp-gcpdisks.data", "]", true);
FileGenerator.writeToFile(DataFileNamesConstants.CLOUD_FUNCTION, "]", true);
FileGenerator.writeToFile(DataFileNamesConstants.CLOUD_FUNCTION_GEN1, "]", true);

Expand Down Expand Up @@ -174,6 +176,9 @@ public static void generateIamUsers(List<IAMUserVH> cloudSqlVHList) {
public static void generateLoadBalancerFiles(List<LoadBalancerVH> fetchLoadBalancerInventory) {
FileGenerator.generateJson(fetchLoadBalancerInventory, "gcp-gcploadbalancer.data");
}
public static void generateDisksFiles(List<DiskVH> fetchDisksList) {
FileGenerator.generateJson(fetchDisksList, "gcp-gcpdisks.data");
}
public static void generateApiKeysFiles(List<APIKeysVH>apiKeysVHList){
FileGenerator.generateJson(apiKeysVHList, "gcp-apikeys.data");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package com.tmobile.pacbot.gcp.inventory.vo;

import com.google.protobuf.ProtocolStringList;

import java.util.List;

public class DiskVH extends GCPVH{

private String name;
private String kind;
private long sizeGb;
private String zone;
private String status;
private String type;
private ProtocolStringList licenses;
private ProtocolStringList users;
private List<Long>licenseCodes;

public String getName() {
return name;
}

public void setName(String name) {
this.name = name;
}

public String getKind() {
return kind;
}

public void setKind(String kind) {
this.kind = kind;
}

public long getSizeGb() {
return sizeGb;
}

public void setSizeGb(long sizeGb) {
this.sizeGb = sizeGb;
}

public String getZone() {
return zone;
}

public void setZone(String zone) {
this.zone = zone;
}

public String getStatus() {
return status;
}

public void setStatus(String status) {
this.status = status;
}

public ProtocolStringList getLicenses() {
return licenses;
}

public void setLicenses(ProtocolStringList licenses) {
this.licenses = licenses;
}

public ProtocolStringList getUsers() {
return users;
}

public void setUsers(ProtocolStringList users) {
this.users = users;
}

public List<Long> getLicenseCodes() {
return licenseCodes;
}

public void setLicenseCodes(List<Long> licenseCodes) {
this.licenseCodes = licenseCodes;
}

public String getType() {
return type;
}

public void setType(String type) {
this.type = type;
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.tmobile.cloud.gcprules.vminstance;
package com.tmobile.cloud.gcprules.Disks;

import com.amazonaws.util.StringUtils;
import com.google.gson.JsonArray;
Expand Down Expand Up @@ -40,7 +40,7 @@ public PolicyResult execute(Map<String, String> ruleParam, Map<String, String> r
}

if (!StringUtils.isNullOrEmpty(vmEsURL)) {
vmEsURL = vmEsURL + "/gcp_vminstance/_search";
vmEsURL = vmEsURL + "/gcp_gcpdisks/_search";
}
logger.debug("========vmEsURL URL after concatenation param {} =========", vmEsURL);
boolean isVMDiskUnused = false;
Expand Down Expand Up @@ -86,23 +86,15 @@ private boolean checkForUnusedDisk(String vmEsURL, Map<String, Object> mustFilte
JsonArray hitsJsonArray = GCPUtils.getHitsArrayFromEs(vmEsURL, mustFilter);
boolean validationResult = false;
if (!hitsJsonArray.isEmpty()) {
JsonObject vmInstanceObject = (JsonObject) ((JsonObject) hitsJsonArray.get(0))
JsonObject disksObject = (JsonObject) ((JsonObject) hitsJsonArray.get(0))
.get(PacmanRuleConstants.SOURCE);

logger.debug("Validating the data item: {}", vmInstanceObject);
logger.debug("Validating the data item: {}", disksObject);

String name=vmInstanceObject.get(PacmanRuleConstants.NAME).getAsString();
JsonArray users=disksObject.get(PacmanRuleConstants.USERS).getAsJsonArray();

if(vmInstanceObject.get(PacmanRuleConstants.DISKS)!=null){
JsonArray disks=vmInstanceObject.get(PacmanRuleConstants.DISKS).getAsJsonArray();
for(int i=0;i<disks.size();i++){
JsonObject diskDataItem = ((JsonObject) disks
.get(i));
String diskName=diskDataItem.get(PacmanRuleConstants.NAME).getAsString();
if(!diskName.equalsIgnoreCase(name)){
validationResult=true;
}
}
if(users.isEmpty()){
validationResult=true;
}

} else {
Expand Down
Loading

0 comments on commit 76ab7b5

Please sign in to comment.