Skip to content

Commit

Permalink
issue #3027 change the way how indices with storage usage is peeking …
Browse files Browse the repository at this point in the history 
…in process of billing calculation (#3110)
  • Loading branch information
@SilinPavel
SilinPavel authored Feb 28, 2023
1 parent ba92c39 commit 0f623f8
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 8 deletions.
9 changes: 9 additions & 0 deletions ...agent/src/main/java/com/epam/pipeline/billingreportagent/app/CommonSyncConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ public class CommonSyncConfiguration {
@Value("${sync.storage.file.index.pattern}")
private String fileIndexPattern;

@Value("${sync.storage.file.alias.index.pattern}")
private String fileAliasIndexPattern;

@Value("${sync.storage.historical.billing.generation:false}")
private boolean enableStorageHistoricalBillingGeneration;

Expand Down Expand Up @@ -139,6 +142,7 @@ public StorageSynchronizer s3Synchronizer(final StorageLoader loader,
new StorageToBillingRequestConverter(mapper, elasticsearchClient,
StorageType.OBJECT_STORAGE,
pricingService,
fileAliasIndexPattern,
fileIndexPattern,
enableStorageHistoricalBillingGeneration),
DataStorageType.S3);
Expand Down Expand Up @@ -170,6 +174,7 @@ public StorageSynchronizer efsSynchronizer(final StorageLoader loader,
new StorageToBillingRequestConverter(mapper, elasticsearchClient,
StorageType.FILE_STORAGE,
pricingService,
fileAliasIndexPattern,
fileIndexPattern,
fileShareMountsService,
MountType.NFS,
Expand All @@ -196,6 +201,7 @@ public StorageSynchronizer gsSynchronizer(final StorageLoader loader,
new StorageToBillingRequestConverter(mapper, elasticsearchClient,
StorageType.OBJECT_STORAGE,
pricingService,
fileAliasIndexPattern,
fileIndexPattern,
enableStorageHistoricalBillingGeneration),
DataStorageType.GS);
Expand Down Expand Up @@ -231,6 +237,7 @@ public StorageSynchronizer azureBlobSynchronizer(
new StorageToBillingRequestConverter(mapper, elasticsearchClient,
StorageType.OBJECT_STORAGE,
pricingService,
fileAliasIndexPattern,
fileIndexPattern,
enableStorageHistoricalBillingGeneration),
DataStorageType.AZ);
Expand Down Expand Up @@ -262,6 +269,7 @@ public StorageSynchronizer azureNetAppSynchronizer(final StorageLoader loader,
new StorageToBillingRequestConverter(mapper, elasticsearchClient,
StorageType.FILE_STORAGE,
pricingService,
fileAliasIndexPattern,
fileIndexPattern,
fileShareMountsService,
MountType.NFS,
Expand Down Expand Up @@ -295,6 +303,7 @@ public StorageSynchronizer azureFilesSynchronizer(final StorageLoader loader,
new StorageToBillingRequestConverter(mapper, elasticsearchClient,
StorageType.FILE_STORAGE,
pricingService,
fileAliasIndexPattern,
fileIndexPattern,
fileShareMountsService,
MountType.SMB,
Expand Down
47 changes: 39 additions & 8 deletions .../pipeline/billingreportagent/service/impl/converter/StorageToBillingRequestConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
Expand Down Expand Up @@ -73,6 +74,7 @@ public class StorageToBillingRequestConverter implements EntityToBillingRequestC
private final ElasticsearchServiceClient elasticsearchService;
private final StorageType storageType;
private final StoragePricingService storagePricing;
private final String esFileAliasIndexPattern;
private final String esFileIndexPattern;
private final Optional<FileShareMountsService> fileshareMountsService;
private final MountType desiredMountType;
Expand All @@ -82,16 +84,18 @@ public StorageToBillingRequestConverter(final AbstractEntityMapper<StorageBillin
final ElasticsearchServiceClient elasticsearchService,
final StorageType storageType,
final StoragePricingService storagePricing,
final String esFileAliasIndexPattern,
final String esFileIndexPattern,
final boolean enableStorageHistoricalBillingGeneration) {
this(mapper, elasticsearchService, storageType, storagePricing, esFileIndexPattern, null, null,
enableStorageHistoricalBillingGeneration);
this(mapper, elasticsearchService, storageType, storagePricing, esFileAliasIndexPattern,
esFileIndexPattern, null, null, enableStorageHistoricalBillingGeneration);
}

public StorageToBillingRequestConverter(final AbstractEntityMapper<StorageBillingInfo> mapper,
final ElasticsearchServiceClient elasticsearchService,
final StorageType storageType,
final StoragePricingService storagePricing,
final String esFileAliasIndexPattern,
final String esFileIndexPattern,
final FileShareMountsService fileshareMountsService,
final MountType desiredMountType,
Expand All @@ -100,6 +104,7 @@ public StorageToBillingRequestConverter(final AbstractEntityMapper<StorageBillin
this.elasticsearchService = elasticsearchService;
this.storageType = storageType;
this.storagePricing = storagePricing;
this.esFileAliasIndexPattern = esFileAliasIndexPattern;
this.esFileIndexPattern = esFileIndexPattern;
this.fileshareMountsService = Optional.ofNullable(fileshareMountsService);
this.desiredMountType = desiredMountType;
Expand Down Expand Up @@ -141,13 +146,10 @@ public List<DocWriteRequest> convertEntitiesToRequests(final List<EntityContaine

private Optional<SearchResponse> requestSumAggregationForStorage(final Long storageId,
final DataStorageType storageType) {
final String searchIndex = String.format(esFileIndexPattern,
storageType.toString().toLowerCase(),
DataStorageType.AZ.equals(storageType) ? "blob" : "file",
storageId);
if (elasticsearchService.isIndexExists(searchIndex)) {
final String indexNameByAlias = getIndexName(storageId, storageType);
if (elasticsearchService.isIndexExists(indexNameByAlias)) {
final SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(searchIndex);
searchRequest.indices(indexNameByAlias);
final TermsAggregationBuilder currentSizeSumAgg = AggregationBuilders
.terms(OBJECT_SIZE_AGG_NAME).field(STORAGE_CLASS_FIELD)
.subAggregation(AggregationBuilders.sum(STORAGE_SIZE_AGG_NAME).field(SIZE_FIELD));
Expand All @@ -164,6 +166,35 @@ private Optional<SearchResponse> requestSumAggregationForStorage(final Long stor
}
}

private String getIndexName(final Long storageId, final DataStorageType storageType) {
final String indexNameByAlias = getIndexNameByAlias(storageId, storageType);
if (indexNameByAlias == null) {
final String indexNamePattern = String.format(esFileIndexPattern, storageType.toString().toLowerCase(),
DataStorageType.AZ.equals(storageType) ? "blob" : "file", storageId);
log.warn("Fail to found index by alias will use name pattern: {} for storage: {}, " +
"billing can be calculated incorrectly!", indexNamePattern, storageId);
return indexNamePattern;
}
return indexNameByAlias;
}

private String getIndexNameByAlias(final Long storageId, final DataStorageType storageType) {
if (esFileAliasIndexPattern == null) {
return null;
}
final String searchIndex = String.format(esFileAliasIndexPattern,
storageType.toString().toLowerCase(),
DataStorageType.AZ.equals(storageType) ? "blob" : "file",
storageId);
try {
return elasticsearchService.getIndexNameByAlias(searchIndex);
} catch (ElasticsearchException e) {
log.warn(String.format("There is a problem to find index for storage %d by alias: %s!",
storageId, searchIndex), e);
return null;
}
}

private List<DocWriteRequest> buildRequestFromAggregation(final EntityContainer<AbstractDataStorage> container,
final LocalDateTime syncStart,
final SearchResponse response,
Expand Down
1 change: 1 addition & 0 deletions billing-report-agent/src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ sync.storage.price.load.mode=json
sync.storage.index.mapping=classpath:/templates/storage_billing.json
sync.storage.index.name=storage-
sync.storage.file.index.pattern=*cp-%s-%s-%d
sync.storage.file.alias.index.pattern=cp-%s-%s-%d
sync.aws.json.price.endpoint.template=https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/%s/current/index.json
sync.storage.billing.exclude.metadata.key=Billing status
sync.storage.billing.exclude.metadata.value=Exclude
8 changes: 8 additions & 0 deletions ...pam/pipeline/billingreportagent/service/impl/converter/StorageToRequestConverterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ public class StorageToRequestConverterTest {
private static final BigDecimal AZ_NFS_STORAGE_1_GB_DAILY_PRICE = BigDecimal.valueOf(2);
public static final String STORAGE_CLASS = "STANDARD";
public static final String ARCHIVE_CLASS = "GLACIER";
public static final String TEST_INDEX_NAME = "test-index-name";

private final PipelineUser testUser = PipelineUser.builder()
.userName(USER_NAME)
Expand Down Expand Up @@ -221,27 +222,31 @@ public void init() {
StorageType.OBJECT_STORAGE,
testStoragePricing,
StringUtils.EMPTY,
StringUtils.EMPTY,
false);
gcpConverter = new StorageToBillingRequestConverter(
new StorageBillingMapper(SearchDocumentType.GS_STORAGE, BILLING_CENTER_KEY),
elasticsearchClient,
StorageType.OBJECT_STORAGE,
testStoragePricing,
StringUtils.EMPTY,
StringUtils.EMPTY,
false);
azureBlobConverter = new StorageToBillingRequestConverter(
new StorageBillingMapper(SearchDocumentType.AZ_BLOB_STORAGE, BILLING_CENTER_KEY),
elasticsearchClient,
StorageType.OBJECT_STORAGE,
testStoragePricing,
StringUtils.EMPTY,
StringUtils.EMPTY,
false);
nfsConverter = new StorageToBillingRequestConverter(
new StorageBillingMapper(SearchDocumentType.NFS_STORAGE, BILLING_CENTER_KEY),
elasticsearchClient,
StorageType.FILE_STORAGE,
testStoragePricing,
StringUtils.EMPTY,
StringUtils.EMPTY,
fileShareMountsService,
MountType.NFS,
false);
Expand All @@ -251,6 +256,7 @@ public void init() {
StorageType.FILE_STORAGE,
testAzureNfsStoragePricing,
StringUtils.EMPTY,
StringUtils.EMPTY,
fileShareMountsService,
MountType.NFS,
false);
Expand All @@ -260,6 +266,7 @@ public void init() {
StorageType.FILE_STORAGE,
testAzureNfsStoragePricing,
StringUtils.EMPTY,
StringUtils.EMPTY,
fileShareMountsService,
MountType.SMB,
false);
Expand Down Expand Up @@ -520,6 +527,7 @@ private void createElasticsearchSearchContext(final boolean isEmptyResponse, fin
Mockito.when(response.getAggregations()).thenReturn(aggregations);
Mockito.when(response.getHits()).thenReturn(hits);
Mockito.when(elasticsearchClient.isIndexExists(Mockito.anyString())).thenReturn(true);
Mockito.when(elasticsearchClient.getIndexNameByAlias(Mockito.anyString())).thenReturn(TEST_INDEX_NAME);
Mockito.when(elasticsearchClient.search(Mockito.any())).thenReturn(response);
}

Expand Down
1 change: 1 addition & 0 deletions deploy/docker/cp-billing-srv/config/application.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ sync.storage.index.mapping=classpath:/templates/storage_billing.json
sync.storage.index.name=storage-
sync.storage.price.load.mode=${CP_BILLING_AWS_PRICE_TYPE:json}
sync.storage.file.index.pattern=${CP_BILLING_STORAGE_INDEX_PATTERN:*cp-%s-%s-%d}
sync.storage.file.alias.index.pattern=${CP_BILLING_STORAGE_ALIAS_INDEX_PATTERN:cp-%s-%s-%d}
sync.aws.json.price.endpoint.template=${CP_BILLING_AWS_PRICES_ENDPOINT:https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/%s/current/index.json}
sync.storage.billing.exclude.metadata.key=${CP_BILLING_STORAGE_EXCLUDE_METADATA_KEY:Billing status}
sync.storage.billing.exclude.metadata.value=${CP_BILLING_STORAGE_EXCLUDE_METADATA_VALUE:Exclude}

0 comments on commit 0f623f8

Please sign in to comment.