From 8b00302c83c125de0ba095d3269bc05a680c65b1 Mon Sep 17 00:00:00 2001 From: Jair Myree Date: Thu, 7 Sep 2023 11:36:32 -0700 Subject: [PATCH] [Monitor Query] Add samples/docs for running big queries and overcoming API limits for size and record count (#35704) * Documentation updates for large queries * Large Query Documentation * Large Query Documentation updates * Large Log Query Documentation * Large Log Query Documentation * Remove additional test * Update sdk/monitor/azure-monitor-query/LargeLogQuery.md Co-authored-by: Liudmila Molkova * Update sdk/monitor/azure-monitor-query/LargeLogQuery.md Co-authored-by: Liudmila Molkova * Update sdk/monitor/azure-monitor-query/LargeLogQuery.md Co-authored-by: Srikanta <51379715+srnagar@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/LargeLogQuery.md Co-authored-by: Srikanta <51379715+srnagar@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/LargeLogQuery.md Co-authored-by: Liudmila Molkova * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/LargeQuerySample.java Co-authored-by: Liudmila Molkova * Update sdk/monitor/azure-monitor-query/LargeLogQuery.md Co-authored-by: Liudmila Molkova * Updating Large Log Query Documentation * Updating Large Query Sample * Updating Large Query Sample PR * Updating Large Query Sample PR * Updating Large Query Sample PR * Updating Large Query Sample PR * Update sdk/monitor/azure-monitor-query/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/OvercomingLargeQueryLimitationsSample.java Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Updating Samples README.md * Updating Samples README.md * Update sdk/monitor/azure-monitor-query/src/samples/java/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/OvercomingLargeQueryLimitationsSample.java Co-authored-by: Srikanta <51379715+srnagar@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/OvercomingLargeQueryLimitationsSample.java Co-authored-by: Srikanta <51379715+srnagar@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/README.md Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/OvercomingLargeQueryLimitationsSample.java Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/OvercomingLargeQueryLimitationsSample.java Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> * Updating the large query documentation PR * Updating the large query documentation PR * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByByteSample.java Co-authored-by: Srikanta <51379715+srnagar@users.noreply.github.com> * Update sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByRowSample.java Co-authored-by: Srikanta <51379715+srnagar@users.noreply.github.com> * Updating the large query documentation PR and adding more documentation to QueryTimeInterval * Updating the large query documentation PR --------- Co-authored-by: Liudmila Molkova Co-authored-by: Srikanta <51379715+srnagar@users.noreply.github.com> Co-authored-by: Scott Addie <10702007+scottaddie@users.noreply.github.com> --- sdk/monitor/azure-monitor-query/README.md | 5 + .../query/models/QueryTimeInterval.java | 1 + .../src/samples/java/README.md | 25 ++- .../query/SplitQueryByByteSizeSample.java | 142 ++++++++++++++++++ .../query/SplitQueryByRowCountSample.java | 127 ++++++++++++++++ 5 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByByteSizeSample.java create mode 100644 sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByRowCountSample.java diff --git a/sdk/monitor/azure-monitor-query/README.md b/sdk/monitor/azure-monitor-query/README.md index 17519efad4a5f..24e7c83416d27 100644 --- a/sdk/monitor/azure-monitor-query/README.md +++ b/sdk/monitor/azure-monitor-query/README.md @@ -444,6 +444,10 @@ raw JSON response. For example: } ``` +#### Overcome Log Analytics query size limitations + +If your query exceeds the [service limits][service_limits], see the large log query documentation to learn how to overcome them. + ### Metrics query A resource ID, as denoted by the `{resource-id}` placeholder in the sample below, is required to query metrics. To find the resource ID: @@ -598,6 +602,7 @@ comments. [samples]: https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/monitor/azure-monitor-query/src/samples/java/README.md [source]: https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/monitor/azure-monitor-query/src [performance_tuning]: https://github.com/Azure/azure-sdk-for-java/wiki/Performance-Tuning +[service_limits]: https://learn.microsoft.com/azure/azure-monitor/service-limits#log-queries-and-language [cla]: https://cla.microsoft.com [coc]: https://opensource.microsoft.com/codeofconduct/ diff --git a/sdk/monitor/azure-monitor-query/src/main/java/com/azure/monitor/query/models/QueryTimeInterval.java b/sdk/monitor/azure-monitor-query/src/main/java/com/azure/monitor/query/models/QueryTimeInterval.java index 0cba434315df3..78f04e02b20df 100644 --- a/sdk/monitor/azure-monitor-query/src/main/java/com/azure/monitor/query/models/QueryTimeInterval.java +++ b/sdk/monitor/azure-monitor-query/src/main/java/com/azure/monitor/query/models/QueryTimeInterval.java @@ -12,6 +12,7 @@ /** * Class to represent a time interval. + * Time intervals are inclusive at the start time and exclusive at the end time. */ @Immutable public final class QueryTimeInterval { diff --git a/sdk/monitor/azure-monitor-query/src/samples/java/README.md b/sdk/monitor/azure-monitor-query/src/samples/java/README.md index 2951758f4c14a..fe63ac258c68c 100644 --- a/sdk/monitor/azure-monitor-query/src/samples/java/README.md +++ b/sdk/monitor/azure-monitor-query/src/samples/java/README.md @@ -23,14 +23,31 @@ Getting started explained in detail [here][SDK_README_GETTING_STARTED]. ## Examples -The following sections provide code samples covering common operations with OpenTelemetry Azure Monitor Exporter client -library. +The following sections provide code samples covering common operations with the Azure Monitor Query client library. * [Get logs for a query][get_logs] * [Get logs for a batch for queries][get_batch_logs] * [Get logs for a query with server timeout][get_servertimeout_logs] * [Get metrics][get_metrics] +## Run large log queries using Log Analytics + +Due to Log Analytics [service limits][monitor_service_limits], sometimes it may +not be possible to retrieve all the expected data in a single query. For example, the number of rows returned or the maximum size of the +data returned may exceed the stated limits. One approach for overcoming these limits is to split the queries into multiple smaller queries +using different time ranges. + +This workaround allows you to avoid the cost of exporting data to a storage account (and potentially the cost of the storage account as well). + +**Disclaimer:** This approach of splitting data retrieval into smaller queries is useful when dealing with a few GBs of data or a few million records per hour. For larger data sets, [exporting][logs_data_export] is recommended. + +We've provided a sample that demonstrates how to split a large query into a batch query based on the number of rows. The sample can be found here. +We've also provided a sample that demonstrates how to split a large query into a batch query based on the size of the data returned. The sample can be found here. + +These sample shows how to partition a large query into smaller queries using the `LogsBatchQuery` class. The partitioning is based on the timestamp "TimeGenerated". + +This sample is suitable for simple data retrieval queries that utilize a subset of KQL operators. The subset of supported KQL operators can be found [here][kql_language_subset]. + ## Troubleshooting Troubleshooting steps can be found [here][SDK_README_TROUBLESHOOTING]. @@ -55,4 +72,6 @@ Guidelines][SDK_README_CONTRIBUTING] for more information. [get_batch_logs]: https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/LogsQueryBatchSample.java [get_servertimeout_logs]: https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/ServerTimeoutSample.java [get_metrics]: https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/MetricsQuerySample.java - +[monitor_service_limits]: https://learn.microsoft.com/azure/azure-monitor/service-limits#la-query-api +[logs_data_export]: https://learn.microsoft.com/azure/azure-monitor/logs/logs-data-export?tabs=portal +[kql_language_subset]: https://learn.microsoft.com/azure/azure-monitor/logs/basic-logs-query?tabs=portal-1#kql-language-limits diff --git a/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByByteSizeSample.java b/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByByteSizeSample.java new file mode 100644 index 0000000000000..f696d80006dc3 --- /dev/null +++ b/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByByteSizeSample.java @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.monitor.query; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.monitor.query.models.LogsBatchQuery; +import com.azure.monitor.query.models.LogsBatchQueryResult; +import com.azure.monitor.query.models.LogsQueryResult; +import com.azure.monitor.query.models.LogsTable; +import com.azure.monitor.query.models.LogsTableCell; +import com.azure.monitor.query.models.LogsTableColumn; +import com.azure.monitor.query.models.LogsTableRow; +import com.azure.monitor.query.models.QueryTimeInterval; + +/** + * A sample to demonstrate using a batch logs query to overcome the service limits for a large query. + */ +public class SplitQueryByByteSizeSample { + + + static String workspaceId; + static LogsQueryClient client; + + /** + * The main method to execute the sample. + * @param args Ignored args. + */ + public static void main(String[] args) { + + String queryString = "AppRequests"; + workspaceId = Configuration.getGlobalConfiguration().get("AZURE_MONITOR_LOGS_WORKSPACE_ID"); + + int maxByteSizePerBatch = 1024 * 1024 * 10; // 10 MB + + client = new LogsQueryClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .buildClient(); + + + // Running a log batch query with a byte size limit on each batch + LogsBatchQuery byteBasedBatchQuery = createBatchQueryFromTimeRanges(queryString, + createQueryTimeIntervalsForBatchQueryByByteSize(queryString, maxByteSizePerBatch)); + + // The result of the byte split query + List byteLimitedResults = client.queryBatch(byteBasedBatchQuery).getBatchResults(); + + // consolidate the results from the batch query + LogsQueryResult combineByteBasedQuery = combineResults(byteLimitedResults); + } + + /** + * Helper method to create a batch query from a query string and a list of time intervals. + * + * @param originalQuery The original query string. + * @param queryTimeIntervals The list of time intervals. + * @return A {@link LogsBatchQuery} object equivalent to the original query. + */ + static LogsBatchQuery createBatchQueryFromTimeRanges(String originalQuery, + List queryTimeIntervals) { + LogsBatchQuery batchQuery = new LogsBatchQuery(); + + for (QueryTimeInterval timeInterval : queryTimeIntervals) { + batchQuery.addWorkspaceQuery(workspaceId, originalQuery, timeInterval); + } + + return batchQuery; + } + + + /** + * Helper method to create a list of time intervals for a batch query based on the byte size limit. + * + * @param originalQuery The original query string. + * @param maxByteSizePerBatch The maximum byte size per batch. If multiple log entries returned in the original + * query have the exact same timestamp, the byte size per batch may exceed this limit. + * @return A list of {@link QueryTimeInterval} objects. + */ + static List createQueryTimeIntervalsForBatchQueryByByteSize(String originalQuery, + int maxByteSizePerBatch) { + /* + * This query finds the start time of each batch extending the query with a batch_num column that determined + * using the estimate_data_size() function. The estimate_data_size() function returns the estimated byte size of + * each row. The batch_num is calculated by dividing the cumulative sum of the byte size of each row by the max + * amount of bytes per row. The batchStart is then calculated by finding the minimum time for each batch_num. + * The batchStart times are then sorted and projected as the result of the query. + */ + String findBatchEndpointsQuery = String.format( + "%1$s | sort by TimeGenerated desc | extend batch_num = row_cumsum(estimate_data_size(*)) / %2$s | summarize batchStart=min(TimeGenerated) by batch_num | sort by batch_num desc | project batchStart", + originalQuery, + maxByteSizePerBatch); + + LogsQueryResult result = client.queryWorkspace(workspaceId, findBatchEndpointsQuery, QueryTimeInterval.ALL); + List rows = result.getTable().getRows(); + List offsetDateTimes = new ArrayList<>(); + List queryTimeIntervals = new ArrayList<>(); + + for (LogsTableRow row : rows) { + row.getColumnValue("batchStart").ifPresent(rowValue -> { + offsetDateTimes.add(rowValue.getValueAsDateTime()); + }); + } + + + for (int i = 0; i < offsetDateTimes.size(); i++) { + OffsetDateTime startTime = offsetDateTimes.get(i); + OffsetDateTime endTime = i == offsetDateTimes.size() - 1 ? OffsetDateTime.now() : offsetDateTimes.get(i + 1); + QueryTimeInterval timeInterval = new QueryTimeInterval(startTime, endTime); + queryTimeIntervals.add(timeInterval); + } + + return queryTimeIntervals; + } + + + /** + * This method simulates the result of a single query from the results of a batch query by combining lists of + * log tables from each batch query result. It is intended only for batch queries resulting from a split single + * query. It is not intended to be used on queries containing statistics, visualization data, or errors. + * @param batchQueryResults The results (lists of log tables) from the split single query. + * @return The result (log tables) in the form of a single query. + */ + static LogsQueryResult combineResults(List batchQueryResults) { + List logsTablesCells = new ArrayList<>(); + List logsTablesRows = new ArrayList<>(); + List logsTablesColumns = new ArrayList<>(); + for (LogsBatchQueryResult batchQueryResult : batchQueryResults) { + for(LogsTable logsTable: batchQueryResult.getAllTables()) { + logsTablesCells.addAll(logsTable.getAllTableCells()); + logsTablesRows.addAll(logsTable.getRows()); + logsTablesColumns.addAll(logsTable.getColumns()); + } + } + return new LogsQueryResult(Collections.singletonList(new LogsTable(logsTablesCells, logsTablesRows, logsTablesColumns)), null, null, null); + } +} diff --git a/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByRowCountSample.java b/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByRowCountSample.java new file mode 100644 index 0000000000000..e4d3bb750c5b0 --- /dev/null +++ b/sdk/monitor/azure-monitor-query/src/samples/java/com/azure/monitor/query/SplitQueryByRowCountSample.java @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.monitor.query; + +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.monitor.query.models.LogsBatchQuery; +import com.azure.monitor.query.models.LogsBatchQueryResult; +import com.azure.monitor.query.models.LogsQueryResult; +import com.azure.monitor.query.models.LogsTable; +import com.azure.monitor.query.models.LogsTableCell; +import com.azure.monitor.query.models.LogsTableColumn; +import com.azure.monitor.query.models.LogsTableRow; +import com.azure.monitor.query.models.QueryTimeInterval; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class SplitQueryByRowCountSample { + + static String workspaceId; + static LogsQueryClient client; + + /** + * The main method to execute the sample. + * @param args Ignored args. + */ + public static void main(String[] args) { + String queryString = "AppRequests"; + workspaceId = Configuration.getGlobalConfiguration().get("AZURE_MONITOR_LOGS_WORKSPACE_ID"); + + // These values are for demonstration purposes only. Please set to the appropriate values for your use case. + int maxRowsPerBatch = 100; // 100 rows. The service maximum is 500,000 rows per query + + client = new LogsQueryClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .buildClient(); + + // Running a log batch query with a row count limit on each batch + LogsBatchQuery rowBasedBatchQuery = createBatchQueryFromTimeRanges(queryString, + createQueryTimeIntervalsForBatchQueryByRowCount(queryString, maxRowsPerBatch)); + + // The result of the row split query + List rowLimitedResults = client.queryBatch(rowBasedBatchQuery).getBatchResults(); + + // consolidate the results from the batch query + LogsQueryResult combineRowBasedQuery = combineResults(rowLimitedResults); + } + + /** + * Helper method to create a batch query from a query string and a list of time intervals. + * + * @param originalQuery The original query string. + * @param queryTimeIntervals The list of time intervals. + * @return A {@link LogsBatchQuery} object equivalent to the original query. + */ + static LogsBatchQuery createBatchQueryFromTimeRanges(String originalQuery, + List queryTimeIntervals) { + LogsBatchQuery batchQuery = new LogsBatchQuery(); + + for (QueryTimeInterval timeInterval : queryTimeIntervals) { + batchQuery.addWorkspaceQuery(workspaceId, originalQuery, timeInterval); + } + + return batchQuery; + } + + /** + * Helper method to create a list of time intervals for a batch query based on the row count limit. + * + * @param originalQuery The original query string. + * @param maxRowPerBatch The maximum row count per batch. If multiple log entries returned in the original query + * have the exact same timestamp, the row count per batch may exceed this limit. + * @return A list of {@link QueryTimeInterval} objects. + */ + static List createQueryTimeIntervalsForBatchQueryByRowCount(String originalQuery, + int maxRowPerBatch) { + + /* + * This query finds the start time of each batch. The batch number is calculated by dividing the cumulative row + * count at each row by the max row count per batch. The batch start time is the minimum time generated for each + * batch number. The batch numbers are then sorted and projected as the result of the query. + */ + String findBatchEndpointsQuery = String.format( + "%1$s | sort by TimeGenerated desc | extend batch_num = row_cumsum(1) / %2$s | summarize batchStart=min(TimeGenerated) by batch_num | sort by batch_num desc | project batchStart", + originalQuery, + maxRowPerBatch); + + LogsQueryResult result = client.queryWorkspace(workspaceId, findBatchEndpointsQuery, QueryTimeInterval.ALL); + List rows = result.getTable().getRows(); + List offsetDateTimes = new ArrayList<>(); + List queryTimeIntervals = new ArrayList<>(); + + for (LogsTableRow row : rows) { + row.getColumnValue("batchStart").ifPresent(rowValue -> { + offsetDateTimes.add(rowValue.getValueAsDateTime()); + }); + } + + + for (int i = 0; i < offsetDateTimes.size(); i++) { + OffsetDateTime startTime = offsetDateTimes.get(i); + OffsetDateTime endTime = i == offsetDateTimes.size() - 1 ? OffsetDateTime.now() : offsetDateTimes.get(i + 1); + QueryTimeInterval timeInterval = new QueryTimeInterval(startTime, endTime); + queryTimeIntervals.add(timeInterval); + } + + return queryTimeIntervals; + } + + static LogsQueryResult combineResults(List batchQueryResults) { + List logsTablesCells = new ArrayList<>(); + List logsTablesRows = new ArrayList<>(); + List logsTablesColumns = new ArrayList<>(); + for (LogsBatchQueryResult batchQueryResult : batchQueryResults) { + for(LogsTable logsTable: batchQueryResult.getAllTables()) { + logsTablesCells.addAll(logsTable.getAllTableCells()); + logsTablesRows.addAll(logsTable.getRows()); + logsTablesColumns.addAll(logsTable.getColumns()); + } + } + return new LogsQueryResult(Collections.singletonList(new LogsTable(logsTablesCells, logsTablesRows, logsTablesColumns)), null, null, null); + } +}