diff --git a/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/app/CommonSyncConfiguration.java b/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/app/CommonSyncConfiguration.java index ede1f177a8..ab437a6a62 100644 --- a/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/app/CommonSyncConfiguration.java +++ b/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/app/CommonSyncConfiguration.java @@ -74,6 +74,9 @@ public class CommonSyncConfiguration { @Value("${sync.storage.file.index.pattern}") private String fileIndexPattern; + @Value("${sync.storage.file.alias.index.pattern}") + private String fileAliasIndexPattern; + @Value("${sync.storage.historical.billing.generation:false}") private boolean enableStorageHistoricalBillingGeneration; @@ -139,6 +142,7 @@ public StorageSynchronizer s3Synchronizer(final StorageLoader loader, new StorageToBillingRequestConverter(mapper, elasticsearchClient, StorageType.OBJECT_STORAGE, pricingService, + fileAliasIndexPattern, fileIndexPattern, enableStorageHistoricalBillingGeneration), DataStorageType.S3); @@ -170,6 +174,7 @@ public StorageSynchronizer efsSynchronizer(final StorageLoader loader, new StorageToBillingRequestConverter(mapper, elasticsearchClient, StorageType.FILE_STORAGE, pricingService, + fileAliasIndexPattern, fileIndexPattern, fileShareMountsService, MountType.NFS, @@ -196,6 +201,7 @@ public StorageSynchronizer gsSynchronizer(final StorageLoader loader, new StorageToBillingRequestConverter(mapper, elasticsearchClient, StorageType.OBJECT_STORAGE, pricingService, + fileAliasIndexPattern, fileIndexPattern, enableStorageHistoricalBillingGeneration), DataStorageType.GS); @@ -231,6 +237,7 @@ public StorageSynchronizer azureBlobSynchronizer( new StorageToBillingRequestConverter(mapper, elasticsearchClient, StorageType.OBJECT_STORAGE, pricingService, + fileAliasIndexPattern, fileIndexPattern, enableStorageHistoricalBillingGeneration), DataStorageType.AZ); @@ -262,6 +269,7 @@ public StorageSynchronizer azureNetAppSynchronizer(final StorageLoader loader, new StorageToBillingRequestConverter(mapper, elasticsearchClient, StorageType.FILE_STORAGE, pricingService, + fileAliasIndexPattern, fileIndexPattern, fileShareMountsService, MountType.NFS, @@ -295,6 +303,7 @@ public StorageSynchronizer azureFilesSynchronizer(final StorageLoader loader, new StorageToBillingRequestConverter(mapper, elasticsearchClient, StorageType.FILE_STORAGE, pricingService, + fileAliasIndexPattern, fileIndexPattern, fileShareMountsService, MountType.SMB, diff --git a/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToBillingRequestConverter.java b/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToBillingRequestConverter.java index 8355fd7adc..2ac2714320 100644 --- a/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToBillingRequestConverter.java +++ b/billing-report-agent/src/main/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToBillingRequestConverter.java @@ -35,6 +35,7 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -73,6 +74,7 @@ public class StorageToBillingRequestConverter implements EntityToBillingRequestC private final ElasticsearchServiceClient elasticsearchService; private final StorageType storageType; private final StoragePricingService storagePricing; + private final String esFileAliasIndexPattern; private final String esFileIndexPattern; private final Optional fileshareMountsService; private final MountType desiredMountType; @@ -82,16 +84,18 @@ public StorageToBillingRequestConverter(final AbstractEntityMapper mapper, final ElasticsearchServiceClient elasticsearchService, final StorageType storageType, final StoragePricingService storagePricing, + final String esFileAliasIndexPattern, final String esFileIndexPattern, final FileShareMountsService fileshareMountsService, final MountType desiredMountType, @@ -100,6 +104,7 @@ public StorageToBillingRequestConverter(final AbstractEntityMapper convertEntitiesToRequests(final List requestSumAggregationForStorage(final Long storageId, final DataStorageType storageType) { - final String searchIndex = String.format(esFileIndexPattern, - storageType.toString().toLowerCase(), - DataStorageType.AZ.equals(storageType) ? "blob" : "file", - storageId); - if (elasticsearchService.isIndexExists(searchIndex)) { + final String indexNameByAlias = getIndexName(storageId, storageType); + if (elasticsearchService.isIndexExists(indexNameByAlias)) { final SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(searchIndex); + searchRequest.indices(indexNameByAlias); final TermsAggregationBuilder currentSizeSumAgg = AggregationBuilders .terms(OBJECT_SIZE_AGG_NAME).field(STORAGE_CLASS_FIELD) .subAggregation(AggregationBuilders.sum(STORAGE_SIZE_AGG_NAME).field(SIZE_FIELD)); @@ -164,6 +166,35 @@ private Optional requestSumAggregationForStorage(final Long stor } } + private String getIndexName(final Long storageId, final DataStorageType storageType) { + final String indexNameByAlias = getIndexNameByAlias(storageId, storageType); + if (indexNameByAlias == null) { + final String indexNamePattern = String.format(esFileIndexPattern, storageType.toString().toLowerCase(), + DataStorageType.AZ.equals(storageType) ? "blob" : "file", storageId); + log.warn("Fail to found index by alias will use name pattern for storage: {}, " + + "billing can be calculated incorrectly!", storageId); + return indexNamePattern; + } + return indexNameByAlias; + } + + private String getIndexNameByAlias(final Long storageId, final DataStorageType storageType) { + if (esFileAliasIndexPattern == null) { + return null; + } + final String searchIndex = String.format(esFileAliasIndexPattern, + storageType.toString().toLowerCase(), + DataStorageType.AZ.equals(storageType) ? "blob" : "file", + storageId); + try { + return elasticsearchService.getIndexNameByAlias(searchIndex); + } catch (ElasticsearchException e) { + log.warn(String.format("There is a problem to find index for storage %d by alias: %s!", + storageId, searchIndex), e); + return null; + } + } + private List buildRequestFromAggregation(final EntityContainer container, final LocalDateTime syncStart, final SearchResponse response, diff --git a/billing-report-agent/src/main/resources/application.properties b/billing-report-agent/src/main/resources/application.properties index c3ab92e271..035916d10c 100644 --- a/billing-report-agent/src/main/resources/application.properties +++ b/billing-report-agent/src/main/resources/application.properties @@ -56,6 +56,7 @@ sync.storage.price.load.mode=json sync.storage.index.mapping=classpath:/templates/storage_billing.json sync.storage.index.name=storage- sync.storage.file.index.pattern=*cp-%s-%s-%d +sync.storage.file.alias.index.pattern=cp-%s-%s-%d sync.aws.json.price.endpoint.template=https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/%s/current/index.json sync.storage.billing.exclude.metadata.key=Billing status sync.storage.billing.exclude.metadata.value=Exclude diff --git a/billing-report-agent/src/test/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToRequestConverterTest.java b/billing-report-agent/src/test/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToRequestConverterTest.java index 0ff35b825c..d4f6685052 100644 --- a/billing-report-agent/src/test/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToRequestConverterTest.java +++ b/billing-report-agent/src/test/java/com/epam/pipeline/billingreportagent/service/impl/converter/StorageToRequestConverterTest.java @@ -165,6 +165,7 @@ public class StorageToRequestConverterTest { private static final BigDecimal AZ_NFS_STORAGE_1_GB_DAILY_PRICE = BigDecimal.valueOf(2); public static final String STORAGE_CLASS = "STANDARD"; public static final String ARCHIVE_CLASS = "GLACIER"; + public static final String TEST_INDEX_NAME = "test-index-name"; private final PipelineUser testUser = PipelineUser.builder() .userName(USER_NAME) @@ -221,6 +222,7 @@ public void init() { StorageType.OBJECT_STORAGE, testStoragePricing, StringUtils.EMPTY, + StringUtils.EMPTY, false); gcpConverter = new StorageToBillingRequestConverter( new StorageBillingMapper(SearchDocumentType.GS_STORAGE, BILLING_CENTER_KEY), @@ -228,6 +230,7 @@ public void init() { StorageType.OBJECT_STORAGE, testStoragePricing, StringUtils.EMPTY, + StringUtils.EMPTY, false); azureBlobConverter = new StorageToBillingRequestConverter( new StorageBillingMapper(SearchDocumentType.AZ_BLOB_STORAGE, BILLING_CENTER_KEY), @@ -235,6 +238,7 @@ public void init() { StorageType.OBJECT_STORAGE, testStoragePricing, StringUtils.EMPTY, + StringUtils.EMPTY, false); nfsConverter = new StorageToBillingRequestConverter( new StorageBillingMapper(SearchDocumentType.NFS_STORAGE, BILLING_CENTER_KEY), @@ -242,6 +246,7 @@ public void init() { StorageType.FILE_STORAGE, testStoragePricing, StringUtils.EMPTY, + StringUtils.EMPTY, fileShareMountsService, MountType.NFS, false); @@ -251,6 +256,7 @@ public void init() { StorageType.FILE_STORAGE, testAzureNfsStoragePricing, StringUtils.EMPTY, + StringUtils.EMPTY, fileShareMountsService, MountType.NFS, false); @@ -260,6 +266,7 @@ public void init() { StorageType.FILE_STORAGE, testAzureNfsStoragePricing, StringUtils.EMPTY, + StringUtils.EMPTY, fileShareMountsService, MountType.SMB, false); @@ -520,6 +527,7 @@ private void createElasticsearchSearchContext(final boolean isEmptyResponse, fin Mockito.when(response.getAggregations()).thenReturn(aggregations); Mockito.when(response.getHits()).thenReturn(hits); Mockito.when(elasticsearchClient.isIndexExists(Mockito.anyString())).thenReturn(true); + Mockito.when(elasticsearchClient.getIndexNameByAlias(Mockito.anyString())).thenReturn(TEST_INDEX_NAME); Mockito.when(elasticsearchClient.search(Mockito.any())).thenReturn(response); } diff --git a/deploy/docker/cp-billing-srv/config/application.properties b/deploy/docker/cp-billing-srv/config/application.properties index 072e2b06f5..4bd893fa8f 100644 --- a/deploy/docker/cp-billing-srv/config/application.properties +++ b/deploy/docker/cp-billing-srv/config/application.properties @@ -56,6 +56,7 @@ sync.storage.index.mapping=classpath:/templates/storage_billing.json sync.storage.index.name=storage- sync.storage.price.load.mode=${CP_BILLING_AWS_PRICE_TYPE:json} sync.storage.file.index.pattern=${CP_BILLING_STORAGE_INDEX_PATTERN:*cp-%s-%s-%d} +sync.storage.file.alias.index.pattern=${CP_BILLING_STORAGE_ALIAS_INDEX_PATTERN:cp-%s-%s-%d} sync.aws.json.price.endpoint.template=${CP_BILLING_AWS_PRICES_ENDPOINT:https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/%s/current/index.json} sync.storage.billing.exclude.metadata.key=${CP_BILLING_STORAGE_EXCLUDE_METADATA_KEY:Billing status} sync.storage.billing.exclude.metadata.value=${CP_BILLING_STORAGE_EXCLUDE_METADATA_VALUE:Exclude}