From b5d6ebf0af03aa8ac2f7dc9ffc18bd7737ef366c Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Tue, 22 Oct 2024 09:07:43 -0700 Subject: [PATCH 1/6] Add simplificiation for SQL iterative includes --- .../QueryGenerators/SqlQueryGenerator.cs | 4 +- .../Features/Search/SqlCommandSimplifier.cs | 164 ++++++++++++++++++ .../Features/Search/SqlServerSearchService.cs | 1 + 3 files changed, 167 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index e229ae7239..126bd55a04 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -944,7 +944,7 @@ private void HandleTableKindInclude( // Handle Multiple Results sets to include from if (count > 1 && _curFromCteIndex >= 0 && _curFromCteIndex < count - 1) { - StringBuilder.AppendLine("),"); + StringBuilder.AppendLine(")").Append(","); // If it's not the last result set, append a new IncludeLimit cte, since IncludeLimitCte was not created for the current cte if (_curFromCteIndex < count - 1) @@ -1156,7 +1156,7 @@ private void WriteIncludeLimitCte(string cteToLimit, SearchOptions context) .AppendLine(" THEN 1 ELSE 0 END AS IsPartial "); StringBuilder.Append("FROM ").AppendLine(cteToLimit); - StringBuilder.AppendLine("),"); + StringBuilder.AppendLine(")").Append(","); // the 'original' include cte is not in the union, but this new layer is instead _includeCteIds.Add(TableExpressionName(_tableExpressionCounter)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs index eee9bd72a6..b29408518f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs @@ -15,6 +15,159 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Search { internal static class SqlCommandSimplifier { + private static Regex s_findCteMatch = new Regex(",cte(\\d+) AS\\r\\n\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = (\\d*)\\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\((\\d*)\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte(\\d+) WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"); + + private static string s_removeCteMatchBase = "(,cte AS\\r\\n\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = \\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\(\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1.*\\r\\n\\s*\\)\\r\\n\\s*,cte AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT .*T1, Sid1, IsMatch, .* AS IsPartial \\r\\n\\s*FROM cte\\r\\n\\s*\\))"; + + private static string s_removeUnionSegmentMatchBase = "(\\s*UNION ALL\\r\\n\\s*SELECT T1, Sid1, IsMatch, IsPartial\\r\\n\\s*FROM cte WHERE NOT EXISTS \\(SELECT \\* FROM cte\\d+ WHERE cte\\d+.Sid1 = cte.Sid1 AND cte\\d+.T1 = cte.T1\\))\\r\\n"; + + private static string s_existsSelectStatement = "SELECT T1, Sid1 FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; + + private static string s_unionExistsSelectStatment = "SELECT T1, Sid1 FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT T1, Sid1 FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; + + internal static void CombineIterativeIncludes(IndentedStringBuilder stringBuilder, ILogger logger) + { + var commandText = stringBuilder.ToString(); + try + { + var cteParams = GetIncludeCteParams(commandText); + + var index = 0; + var redundantCtes = new List<(ReferenceSearchInformation, ReferenceSearchInformation)>(); + var unionCanidateCtes = new List<(ReferenceSearchInformation, ReferenceSearchInformation)>(); + foreach (var item in cteParams) + { + for (int i = index + 1; i < cteParams.Count; i++) + { + if (item.SearchParamId == cteParams[i].SearchParamId && item.ResourceTypeId == cteParams[i].ResourceTypeId) + { + if (!redundantCtes.Exists((redundantCte) => + (redundantCte.Item1.CteNumber == item.SourceCte && redundantCte.Item2.CteNumber == cteParams[i].SourceCte) + || (redundantCte.Item2.CteNumber == item.SourceCte && redundantCte.Item1.CteNumber == cteParams[i].SourceCte))) + { + unionCanidateCtes.Add((item, cteParams[i])); + } + else + { + redundantCtes.Add((item, cteParams[i])); + } + } + } + } + + foreach (var redundantCte in redundantCtes) + { + // Removes the iterate cte and its projection cte, and cleans up the union at the end. + commandText = RemoveCtePair(commandText, redundantCte.Item1); + } + + foreach (var unionCanidate in unionCanidateCtes) + { + // Unions the two ctes together and removes the first one. + commandText = UnionCtes(commandText, unionCanidate); + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Exception combining iterative includes."); + return; // Use the unmodified string + } + + stringBuilder.Clear(); + stringBuilder.Append(commandText); + } + + private static List GetIncludeCteParams(string commandText) + { + var ctes = new List(); + var cteMatches = s_findCteMatch.Matches(commandText); + + foreach (Match match in cteMatches) + { + ctes.Add(new ReferenceSearchInformation() + { + SearchParamId = int.Parse(match.Groups[2].Value), + ResourceTypeId = int.Parse(match.Groups[3].Value), + SourceCte = int.Parse(match.Groups[4].Value), + CteNumber = int.Parse(match.Groups[1].Value), + }); + } + + return ctes; + } + + private static string RemoveCtePair(string commandText, ReferenceSearchInformation cteInfo) + { + var removeCteMatch = s_removeCteMatchBase + .Replace("", cteInfo.CteNumber.ToString(), StringComparison.Ordinal) + .Replace("", cteInfo.SearchParamId.ToString(), StringComparison.Ordinal) + .Replace("", cteInfo.ResourceTypeId.ToString(), StringComparison.Ordinal) + .Replace("", cteInfo.SourceCte.ToString(), StringComparison.Ordinal) + .Replace("", (cteInfo.CteNumber + 1).ToString(), StringComparison.Ordinal); + var removeCteRegex = new Regex(removeCteMatch); + var matches = removeCteRegex.Matches(commandText); + + if (matches.Count > 1) + { + throw new ArgumentException("More than one match found for cte to remove"); + } + else if (matches.Count == 0) + { + throw new ArgumentException("No matches found for cte to remove"); + } + + commandText = commandText.Remove(commandText.IndexOf(matches[0].Value, StringComparison.Ordinal), matches[0].Value.Length); + + var removeUnionSegmentMatch = s_removeUnionSegmentMatchBase + .Replace("", (cteInfo.CteNumber + 1).ToString(), StringComparison.Ordinal); + var removeUnionSegmentRegex = new Regex(removeUnionSegmentMatch); + var unionSegmentMatches = removeUnionSegmentRegex.Matches(commandText); + + if (unionSegmentMatches.Count > 1) + { + throw new ArgumentException("More than one match found for union segment to remove"); + } + else if (unionSegmentMatches.Count == 0) + { + throw new ArgumentException("No matches found for union segment to remove"); + } + + commandText = commandText.Remove(commandText.IndexOf(unionSegmentMatches[0].Value, StringComparison.Ordinal), unionSegmentMatches[0].Value.Length); + + return commandText; + } + + private static string UnionCtes(string commandText, (ReferenceSearchInformation can1, ReferenceSearchInformation can2) unionCanidate) + { + var unionCteMatch = s_removeCteMatchBase + .Replace("", unionCanidate.can2.CteNumber.ToString(), StringComparison.Ordinal) + .Replace("", unionCanidate.can2.SearchParamId.ToString(), StringComparison.Ordinal) + .Replace("", unionCanidate.can2.ResourceTypeId.ToString(), StringComparison.Ordinal) + .Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal) + .Replace("", (unionCanidate.can2.CteNumber + 1).ToString(), StringComparison.Ordinal); + var unionCteRegex = new Regex(unionCteMatch); + var matches = unionCteRegex.Matches(commandText); + + if (matches.Count > 1) + { + throw new ArgumentException("More than one match found for union cte"); + } + else if (matches.Count == 0) + { + throw new ArgumentException("No matches found for union cte"); + } + + var existsSelectStatement = s_existsSelectStatement.Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal); + var newExistsSelectStatement = s_unionExistsSelectStatment + .Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal) + .Replace("", unionCanidate.can1.SourceCte.ToString(), StringComparison.Ordinal); + var newCte = matches[0].Value.Replace(existsSelectStatement, newExistsSelectStatement, StringComparison.Ordinal); + commandText = commandText.Replace(matches[0].Value, newCte, StringComparison.Ordinal); + + commandText = RemoveCtePair(commandText, unionCanidate.can1); + return commandText; + } + internal static void RemoveRedundantParameters(IndentedStringBuilder stringBuilder, SqlParameterCollection sqlParameterCollection, ILogger logger) { var commandText = stringBuilder.ToString(); @@ -95,5 +248,16 @@ private static string RemoveRedundantComparisons(string commandText, SqlParamete return commandText; } + + private class ReferenceSearchInformation + { + public int CteNumber { get; set; } + + public int SearchParamId { get; set; } + + public int ResourceTypeId { get; set; } + + public int SourceCte { get; set; } + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index edaf6a440c..3f448bc73d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -362,6 +362,7 @@ await _sqlRetryService.ExecuteSql( expression.AcceptVisitor(queryGenerator, clonedSearchOptions); SqlCommandSimplifier.RemoveRedundantParameters(stringBuilder, sqlCommand.Parameters, _logger); + SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); var queryText = stringBuilder.ToString(); var queryHash = _queryHashCalculator.CalculateHash(queryText); From 92a94d8cfc29253aa338283fa4cd5e9450b9afeb Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Thu, 24 Oct 2024 11:02:28 -0700 Subject: [PATCH 2/6] Finish adding include iterate simplification. --- .../Features/Search/SqlCommandSimplifier.cs | 18 +- .../Features/Search/SqlServerSearchService.cs | 409 +++++++++--------- 2 files changed, 225 insertions(+), 202 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs index b29408518f..49a0e0a8f1 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs @@ -15,15 +15,15 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Search { internal static class SqlCommandSimplifier { - private static Regex s_findCteMatch = new Regex(",cte(\\d+) AS\\r\\n\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = (\\d*)\\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\((\\d*)\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte(\\d+) WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"); + private static Regex s_findCteMatch = new Regex(",cte(\\d+) AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = (\\d*)\\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\((\\d*)\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte(\\d+) WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"); - private static string s_removeCteMatchBase = "(,cte AS\\r\\n\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = \\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\(\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1.*\\r\\n\\s*\\)\\r\\n\\s*,cte AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT .*T1, Sid1, IsMatch, .* AS IsPartial \\r\\n\\s*FROM cte\\r\\n\\s*\\))"; + private static string s_removeCteMatchBase = "(\\s*,cte AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = \\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\(\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1.*\\r\\n\\s*\\)\\r\\n\\s*,cte AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT .*T1, Sid1, IsMatch, .* AS IsPartial \\r\\n\\s*FROM cte\\r\\n\\s*\\))"; - private static string s_removeUnionSegmentMatchBase = "(\\s*UNION ALL\\r\\n\\s*SELECT T1, Sid1, IsMatch, IsPartial\\r\\n\\s*FROM cte WHERE NOT EXISTS \\(SELECT \\* FROM cte\\d+ WHERE cte\\d+.Sid1 = cte.Sid1 AND cte\\d+.T1 = cte.T1\\))\\r\\n"; + private static string s_removeUnionSegmentMatchBase = "(\\s*UNION ALL\\r\\n\\s*SELECT T1, Sid1, IsMatch, IsPartial\\r\\n\\s*FROM cte WHERE NOT EXISTS \\(SELECT \\* FROM cte\\d+ WHERE cte\\d+.Sid1 = cte.Sid1 AND cte\\d+.T1 = cte.T1\\))"; - private static string s_existsSelectStatement = "SELECT T1, Sid1 FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; + private static string s_existsSelectStatement = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; - private static string s_unionExistsSelectStatment = "SELECT T1, Sid1 FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT T1, Sid1 FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; + private static string s_unionExistsSelectStatment = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; internal static void CombineIterativeIncludes(IndentedStringBuilder stringBuilder, ILogger logger) { @@ -41,9 +41,9 @@ internal static void CombineIterativeIncludes(IndentedStringBuilder stringBuilde { if (item.SearchParamId == cteParams[i].SearchParamId && item.ResourceTypeId == cteParams[i].ResourceTypeId) { - if (!redundantCtes.Exists((redundantCte) => - (redundantCte.Item1.CteNumber == item.SourceCte && redundantCte.Item2.CteNumber == cteParams[i].SourceCte) - || (redundantCte.Item2.CteNumber == item.SourceCte && redundantCte.Item1.CteNumber == cteParams[i].SourceCte))) + if (!unionCanidateCtes.Exists((unionCte) => + (unionCte.Item1.CteNumber + 1 == item.SourceCte && unionCte.Item2.CteNumber + 1 == cteParams[i].SourceCte) + || (unionCte.Item2.CteNumber + 1 == item.SourceCte && unionCte.Item1.CteNumber + 1 == cteParams[i].SourceCte))) { unionCanidateCtes.Add((item, cteParams[i])); } @@ -53,6 +53,8 @@ internal static void CombineIterativeIncludes(IndentedStringBuilder stringBuilde } } } + + index++; } foreach (var redundantCte in redundantCtes) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 3f448bc73d..a472422e7f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -224,7 +224,7 @@ public override async Task SearchAsync(SearchOptions searchOptions return searchResult; } - private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, CancellationToken cancellationToken) + private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, CancellationToken cancellationToken, bool skipSimplification = false) { Expression searchExpression = sqlSearchOptions.Expression; @@ -332,246 +332,267 @@ private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, C await CreateStats(expression, cancellationToken); SearchResult searchResult = null; - await _sqlRetryService.ExecuteSql( - async (connection, cancellationToken, sqlException) => - { - using (SqlCommand sqlCommand = connection.CreateCommand()) // WARNING, this code will not set sqlCommand.Transaction. Sql transactions via C#/.NET are not supported in this method. - { - sqlCommand.CommandTimeout = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; + var commandSimplified = false; - var exportTimeTravel = clonedSearchOptions.QueryHints != null && ContainsGlobalEndSurrogateId(clonedSearchOptions); - if (exportTimeTravel) - { - PopulateSqlCommandFromQueryHints(clonedSearchOptions, sqlCommand); - sqlCommand.CommandTimeout = 1200; // set to 20 minutes, as dataset is usually large - } - else + try + { + await _sqlRetryService.ExecuteSql( + async (connection, cancellationToken, sqlException) => + { + using (SqlCommand sqlCommand = connection.CreateCommand()) // WARNING, this code will not set sqlCommand.Transaction. Sql transactions via C#/.NET are not supported in this method. { - var stringBuilder = new IndentedStringBuilder(new StringBuilder()); - - EnableTimeAndIoMessageLogging(stringBuilder, connection); - - var queryGenerator = new SqlQueryGenerator( - stringBuilder, - new HashingSqlQueryParameterManager(new SqlQueryParameterManager(sqlCommand.Parameters)), - _model, - _schemaInformation, - _reuseQueryPlans.IsEnabled(_sqlRetryService), - sqlException); - - expression.AcceptVisitor(queryGenerator, clonedSearchOptions); - - SqlCommandSimplifier.RemoveRedundantParameters(stringBuilder, sqlCommand.Parameters, _logger); - SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); + sqlCommand.CommandTimeout = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; - var queryText = stringBuilder.ToString(); - var queryHash = _queryHashCalculator.CalculateHash(queryText); - _logger.LogInformation("SQL Search Service query hash: {QueryHash}", queryHash); - var customQuery = CustomQueries.CheckQueryHash(connection, queryHash, _logger); - - if (!string.IsNullOrEmpty(customQuery)) + var exportTimeTravel = clonedSearchOptions.QueryHints != null && ContainsGlobalEndSurrogateId(clonedSearchOptions); + if (exportTimeTravel) { - _logger.LogInformation("SQl Search Service, custom Query identified by hash {QueryHash}, {CustomQuery}", queryHash, customQuery); - queryText = customQuery; - sqlCommand.CommandType = CommandType.StoredProcedure; + PopulateSqlCommandFromQueryHints(clonedSearchOptions, sqlCommand); + sqlCommand.CommandTimeout = 1200; // set to 20 minutes, as dataset is usually large } + else + { + var stringBuilder = new IndentedStringBuilder(new StringBuilder()); - // Command text contains no direct user input. -#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities - sqlCommand.CommandText = queryText; -#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities - } + EnableTimeAndIoMessageLogging(stringBuilder, connection); - LogSqlCommand(sqlCommand); + var queryGenerator = new SqlQueryGenerator( + stringBuilder, + new HashingSqlQueryParameterManager(new SqlQueryParameterManager(sqlCommand.Parameters)), + _model, + _schemaInformation, + _reuseQueryPlans.IsEnabled(_sqlRetryService), + sqlException); - using (var reader = await sqlCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken)) - { - if (clonedSearchOptions.CountOnly) - { - await reader.ReadAsync(cancellationToken); - long count = reader.GetInt64(0); - if (count > int.MaxValue) - { - _requestContextAccessor.RequestContext.BundleIssues.Add( - new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.NotSupported, - string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue))); + expression.AcceptVisitor(queryGenerator, clonedSearchOptions); - _logger.LogWarning("Invalid Search Operation (SearchCountResultsExceedLimit)"); - throw new InvalidSearchOperationException(string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); + if (!skipSimplification) + { + var originonalCommandText = stringBuilder.ToString(); + SqlCommandSimplifier.RemoveRedundantParameters(stringBuilder, sqlCommand.Parameters, _logger); + SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); + commandSimplified = originonalCommandText != stringBuilder.ToString(); } - searchResult = new SearchResult((int)count, clonedSearchOptions.UnsupportedSearchParams); + var queryText = stringBuilder.ToString(); + var queryHash = _queryHashCalculator.CalculateHash(queryText); + _logger.LogInformation("SQL Search Service query hash: {QueryHash}", queryHash); + var customQuery = CustomQueries.CheckQueryHash(connection, queryHash, _logger); - // call NextResultAsync to get the info messages - await reader.NextResultAsync(cancellationToken); + if (!string.IsNullOrEmpty(customQuery)) + { + _logger.LogInformation("SQl Search Service, custom Query identified by hash {QueryHash}, {CustomQuery}", queryHash, customQuery); + queryText = customQuery; + sqlCommand.CommandType = CommandType.StoredProcedure; + } - return; + // Command text contains no direct user input. +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + sqlCommand.CommandText = queryText; +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities } - var resources = new List(sqlSearchOptions.MaxItemCount); - short? newContinuationType = null; - long? newContinuationId = null; - bool moreResults = false; - int matchCount = 0; - - string sortValue = null; - var isResultPartial = false; - int numberOfColumnsRead = 0; + LogSqlCommand(sqlCommand); - while (await reader.ReadAsync(cancellationToken)) + using (var reader = await sqlCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken)) { - ReadWrapper( - reader, - out short resourceTypeId, - out string resourceId, - out int version, - out bool isDeleted, - out long resourceSurrogateId, - out string requestMethod, - out bool isMatch, - out bool isPartialEntry, - out bool isRawResourceMetaSet, - out string searchParameterHash, - out byte[] rawResourceBytes, - out bool isInvisible); - - if (isInvisible) + if (clonedSearchOptions.CountOnly) { - continue; - } + await reader.ReadAsync(cancellationToken); + long count = reader.GetInt64(0); + if (count > int.MaxValue) + { + _requestContextAccessor.RequestContext.BundleIssues.Add( + new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.NotSupported, + string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue))); + + _logger.LogWarning("Invalid Search Operation (SearchCountResultsExceedLimit)"); + throw new InvalidSearchOperationException(string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); + } - numberOfColumnsRead = reader.FieldCount; + searchResult = new SearchResult((int)count, clonedSearchOptions.UnsupportedSearchParams); - // If we get to this point, we know there are more results so we need a continuation token - // Additionally, this resource shouldn't be included in the results - if (matchCount >= clonedSearchOptions.MaxItemCount && isMatch) - { - moreResults = true; + // call NextResultAsync to get the info messages + await reader.NextResultAsync(cancellationToken); - continue; + return; } - Lazy rawResource = new Lazy(() => string.Empty); + var resources = new List(sqlSearchOptions.MaxItemCount); + short? newContinuationType = null; + long? newContinuationId = null; + bool moreResults = false; + int matchCount = 0; - if (!clonedSearchOptions.OnlyIds) + string sortValue = null; + var isResultPartial = false; + int numberOfColumnsRead = 0; + + while (await reader.ReadAsync(cancellationToken)) { - rawResource = new Lazy(() => + ReadWrapper( + reader, + out short resourceTypeId, + out string resourceId, + out int version, + out bool isDeleted, + out long resourceSurrogateId, + out string requestMethod, + out bool isMatch, + out bool isPartialEntry, + out bool isRawResourceMetaSet, + out string searchParameterHash, + out byte[] rawResourceBytes, + out bool isInvisible); + + if (isInvisible) { - using var rawResourceStream = new MemoryStream(rawResourceBytes); - var decompressedResource = _compressedRawResourceConverter.ReadCompressedRawResource(rawResourceStream); + continue; + } - _logger.LogVerbose(_parameterStore, cancellationToken, "{NameOfResourceSurrogateId}: {ResourceSurrogateId}; {NameOfResourceTypeId}: {ResourceTypeId}; Decompressed length: {RawResourceLength}", nameof(resourceSurrogateId), resourceSurrogateId, nameof(resourceTypeId), resourceTypeId, decompressedResource.Length); + numberOfColumnsRead = reader.FieldCount; - if (string.IsNullOrEmpty(decompressedResource)) - { - decompressedResource = MissingResourceFactory.CreateJson(resourceId, _model.GetResourceTypeName(resourceTypeId), "warning", "incomplete"); - _requestContextAccessor.SetMissingResourceCode(System.Net.HttpStatusCode.PartialContent); - } + // If we get to this point, we know there are more results so we need a continuation token + // Additionally, this resource shouldn't be included in the results + if (matchCount >= clonedSearchOptions.MaxItemCount && isMatch) + { + moreResults = true; - return decompressedResource; - }); - } + continue; + } - // See if this resource is a continuation token candidate and increase the count - if (isMatch) - { - newContinuationType = resourceTypeId; - newContinuationId = resourceSurrogateId; + Lazy rawResource = new Lazy(() => string.Empty); - // For normal queries, we select _defaultNumberOfColumnsReadFromResult number of columns. - // If we have more, that means we have an extra column tracking sort value. - // Keep track of sort value if this is the last row. - if (matchCount == clonedSearchOptions.MaxItemCount - 1 && reader.FieldCount > _defaultNumberOfColumnsReadFromResult) + if (!clonedSearchOptions.OnlyIds) { - var tempSortValue = reader.GetValue(SortValueColumnName); - if ((tempSortValue as DateTime?) != null) + rawResource = new Lazy(() => { - sortValue = (tempSortValue as DateTime?).Value.ToString("o"); - } - else + using var rawResourceStream = new MemoryStream(rawResourceBytes); + var decompressedResource = _compressedRawResourceConverter.ReadCompressedRawResource(rawResourceStream); + + _logger.LogVerbose(_parameterStore, cancellationToken, "{NameOfResourceSurrogateId}: {ResourceSurrogateId}; {NameOfResourceTypeId}: {ResourceTypeId}; Decompressed length: {RawResourceLength}", nameof(resourceSurrogateId), resourceSurrogateId, nameof(resourceTypeId), resourceTypeId, decompressedResource.Length); + + if (string.IsNullOrEmpty(decompressedResource)) + { + decompressedResource = MissingResourceFactory.CreateJson(resourceId, _model.GetResourceTypeName(resourceTypeId), "warning", "incomplete"); + _requestContextAccessor.SetMissingResourceCode(System.Net.HttpStatusCode.PartialContent); + } + + return decompressedResource; + }); + } + + // See if this resource is a continuation token candidate and increase the count + if (isMatch) + { + newContinuationType = resourceTypeId; + newContinuationId = resourceSurrogateId; + + // For normal queries, we select _defaultNumberOfColumnsReadFromResult number of columns. + // If we have more, that means we have an extra column tracking sort value. + // Keep track of sort value if this is the last row. + if (matchCount == clonedSearchOptions.MaxItemCount - 1 && reader.FieldCount > _defaultNumberOfColumnsReadFromResult) { - sortValue = tempSortValue.ToString(); + var tempSortValue = reader.GetValue(SortValueColumnName); + if ((tempSortValue as DateTime?) != null) + { + sortValue = (tempSortValue as DateTime?).Value.ToString("o"); + } + else + { + sortValue = tempSortValue.ToString(); + } } + + matchCount++; } - matchCount++; + // as long as at least one entry was marked as partial, this resultset + // should be marked as partial + isResultPartial = isResultPartial || isPartialEntry; + + resources.Add(new SearchResultEntry( + new ResourceWrapper( + resourceId, + version.ToString(CultureInfo.InvariantCulture), + _model.GetResourceTypeName(resourceTypeId), + clonedSearchOptions.OnlyIds ? null : new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet), + new ResourceRequest(requestMethod), + resourceSurrogateId.ToLastUpdated(), + isDeleted, + null, + null, + null, + searchParameterHash, + resourceSurrogateId), + isMatch ? SearchEntryMode.Match : SearchEntryMode.Include)); } - // as long as at least one entry was marked as partial, this resultset - // should be marked as partial - isResultPartial = isResultPartial || isPartialEntry; - - resources.Add(new SearchResultEntry( - new ResourceWrapper( - resourceId, - version.ToString(CultureInfo.InvariantCulture), - _model.GetResourceTypeName(resourceTypeId), - clonedSearchOptions.OnlyIds ? null : new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet), - new ResourceRequest(requestMethod), - resourceSurrogateId.ToLastUpdated(), - isDeleted, - null, - null, - null, - searchParameterHash, - resourceSurrogateId), - isMatch ? SearchEntryMode.Match : SearchEntryMode.Include)); - } + // call NextResultAsync to get the info messages + await reader.NextResultAsync(cancellationToken); - // call NextResultAsync to get the info messages - await reader.NextResultAsync(cancellationToken); + ContinuationToken continuationToken = moreResults + ? new ContinuationToken( + clonedSearchOptions.Sort.Select(s => + s.searchParameterInfo.Name switch + { + SearchParameterNames.ResourceType => (object)newContinuationType, + SearchParameterNames.LastUpdated => newContinuationId, + _ => sortValue, + }).ToArray()) + : null; - ContinuationToken continuationToken = moreResults - ? new ContinuationToken( - clonedSearchOptions.Sort.Select(s => - s.searchParameterInfo.Name switch - { - SearchParameterNames.ResourceType => (object)newContinuationType, - SearchParameterNames.LastUpdated => newContinuationId, - _ => sortValue, - }).ToArray()) - : null; + if (isResultPartial) + { + _logger.LogWarning("Bundle Partial Result (TruncatedIncludeMessage)"); + _requestContextAccessor.RequestContext.BundleIssues.Add( + new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Warning, + OperationOutcomeConstants.IssueType.Incomplete, + Core.Resources.TruncatedIncludeMessage)); + } - if (isResultPartial) - { - _logger.LogWarning("Bundle Partial Result (TruncatedIncludeMessage)"); - _requestContextAccessor.RequestContext.BundleIssues.Add( - new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Warning, - OperationOutcomeConstants.IssueType.Incomplete, - Core.Resources.TruncatedIncludeMessage)); - } + // If this is a sort query, lets keep track of whether we actually searched for sort values. + if (clonedSearchOptions.Sort != null && + clonedSearchOptions.Sort.Count > 0 && + clonedSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated) + { + // If there is an extra column for sort value, we know we have searched for sort values. If no results were returned, we don't know if we have searched for sort values so we need to assume we did so we run the second phase. + sqlSearchOptions.DidWeSearchForSortValue = numberOfColumnsRead > _defaultNumberOfColumnsReadFromResult; + } - // If this is a sort query, lets keep track of whether we actually searched for sort values. - if (clonedSearchOptions.Sort != null && - clonedSearchOptions.Sort.Count > 0 && - clonedSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated) - { - // If there is an extra column for sort value, we know we have searched for sort values. If no results were returned, we don't know if we have searched for sort values so we need to assume we did so we run the second phase. - sqlSearchOptions.DidWeSearchForSortValue = numberOfColumnsRead > _defaultNumberOfColumnsReadFromResult; - } + // This value is set inside the SortRewriter. If it is set, we need to pass + // this value back to the caller. + if (clonedSearchOptions.IsSortWithFilter) + { + sqlSearchOptions.IsSortWithFilter = true; + } - // This value is set inside the SortRewriter. If it is set, we need to pass - // this value back to the caller. - if (clonedSearchOptions.IsSortWithFilter) - { - sqlSearchOptions.IsSortWithFilter = true; - } + if (clonedSearchOptions.SortHasMissingModifier) + { + sqlSearchOptions.SortHasMissingModifier = true; + } - if (clonedSearchOptions.SortHasMissingModifier) - { - sqlSearchOptions.SortHasMissingModifier = true; + searchResult = new SearchResult(resources, continuationToken?.ToJson(), originalSort, clonedSearchOptions.UnsupportedSearchParams); } - - searchResult = new SearchResult(resources, continuationToken?.ToJson(), originalSort, clonedSearchOptions.UnsupportedSearchParams); } - } - }, - _logger, - cancellationToken, - true); // this enables reads from replicas + }, + _logger, + cancellationToken, + true); // this enables reads from replicas + } + catch (SqlException ex) + { + if (commandSimplified) + { + _logger.LogWarning(ex, "Error executing simplified command. Retrying with original command."); + return await SearchImpl(sqlSearchOptions, cancellationToken, true); + } + + throw; + } + return searchResult; } From 9cf2e2ffce32f9631c8c0cc7474cccf605ae3189 Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Tue, 29 Oct 2024 11:13:25 -0700 Subject: [PATCH 3/6] Add unit tests --- .../Search/SqlCommandSimplifierTests.cs | 34 +++++++++++++++++++ .../Features/Search/SqlCommandSimplifier.cs | 22 ++++++------ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs index 2bfd278dd0..46c0a07b0c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs @@ -21,6 +21,12 @@ public class SqlCommandSimplifierTests { private ILogger _logger; + private readonly string _iterativeIncludeStartingQuery = "DECLARE @p0 int = 11\r\nDECLARE @p1 int = 11\r\nDECLARE @p2 int = 100\r\nDECLARE @p3 int = 100\r\nDECLARE @p4 int = 100\r\nDECLARE @p5 int = 100\r\n\r\nSET STATISTICS IO ON;\r\nSET STATISTICS TIME ON;\r\n\r\nDECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)\r\n;WITH\r\ncte0 AS\r\n(\r\n SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1\r\n FROM dbo.Resource\r\n WHERE IsHistory = 0\r\n\t AND IsDeleted = 0\r\n\t AND ResourceTypeId = 40\r\n)\r\n,cte1 AS\r\n(\r\n SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *\r\n FROM\r\n (\r\n\t SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial\r\n\t FROM cte0\r\n\t ORDER BY T1 ASC, Sid1 ASC\r\n ) t\r\n)\r\nINSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1\r\n;WITH cte1 AS (SELECT * FROM @FilteredData)\r\n,cte2 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)\r\n)\r\n,cte3 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte2\r\n)\r\n,cte4 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 404\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte5 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte4\r\n)\r\n,cte6 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (124)\r\n\t AND EXISTS (SELECT * FROM cte5 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte7 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte6\r\n)\r\n,cte8 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte9 AS\r\n(\r\n SELECT DISTINCT TOP (@p2) T1, Sid1, IsMatch, CASE WHEN count_big(*) over() > @p3 THEN 1 ELSE 0 END AS IsPartial\r\n FROM cte8\r\n)\r\n,cte10 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte11 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte10\r\n)\r\n,cte12 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte9 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte13 AS\r\n(\r\n SELECT DISTINCT TOP (@p4) T1, Sid1, IsMatch, CASE WHEN count_big(*) over() > @p5 THEN 1 ELSE 0 END AS IsPartial\r\n FROM cte12\r\n)\r\n,cte14 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte11 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte15 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte14\r\n)\r\n,cte16 AS\r\n(\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte1\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte9 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte9.Sid1 AND cte1.T1 = cte9.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte11 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte11.Sid1 AND cte1.T1 = cte11.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte13 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte13.Sid1 AND cte1.T1 = cte13.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte15 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte15.Sid1 AND cte1.T1 = cte15.T1)\r\n)\r\nSELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource\r\nFROM dbo.Resource r\r\n JOIN cte16 ON r.ResourceTypeId = cte16.T1 AND r.ResourceSurrogateId = cte16.Sid1\r\nWHERE IsHistory = 0\r\n AND IsDeleted = 0\r\nORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC\r\n\r\nOPTION (RECOMPILE)\r\n-- execution timeout = 180 sec."; + + private readonly string _iterativeIncludeEndingQuery = "DECLARE @p0 int = 11\r\nDECLARE @p1 int = 11\r\nDECLARE @p2 int = 100\r\nDECLARE @p3 int = 100\r\nDECLARE @p4 int = 100\r\nDECLARE @p5 int = 100\r\n\r\nSET STATISTICS IO ON;\r\nSET STATISTICS TIME ON;\r\n\r\nDECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)\r\n;WITH\r\ncte0 AS\r\n(\r\n SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1\r\n FROM dbo.Resource\r\n WHERE IsHistory = 0\r\n\t AND IsDeleted = 0\r\n\t AND ResourceTypeId = 40\r\n)\r\n,cte1 AS\r\n(\r\n SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *\r\n FROM\r\n (\r\n\t SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial\r\n\t FROM cte0\r\n\t ORDER BY T1 ASC, Sid1 ASC\r\n ) t\r\n)\r\nINSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1\r\n;WITH cte1 AS (SELECT * FROM @FilteredData)\r\n,cte2 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)\r\n)\r\n,cte3 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte2\r\n)\r\n,cte4 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 404\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte5 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte4\r\n)\r\n,cte6 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (124)\r\n\t AND EXISTS (SELECT * FROM cte5 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte7 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte6\r\n)\r\n,cte10 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte11 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte10\r\n)\r\n,cte14 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte11 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte15 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte14\r\n)\r\n,cte16 AS\r\n(\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte1\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte11 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte11.Sid1 AND cte1.T1 = cte11.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte15 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte15.Sid1 AND cte1.T1 = cte15.T1)\r\n)\r\nSELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource\r\nFROM dbo.Resource r\r\n JOIN cte16 ON r.ResourceTypeId = cte16.T1 AND r.ResourceSurrogateId = cte16.Sid1\r\nWHERE IsHistory = 0\r\n AND IsDeleted = 0\r\nORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC\r\n\r\nOPTION (RECOMPILE)\r\n-- execution timeout = 180 sec."; + + private readonly string _iterativeIncludeUnchangedQuery = "DECLARE @p0 int = 11\r\nDECLARE @p1 int = 11\r\n\r\nSET STATISTICS IO ON;\r\nSET STATISTICS TIME ON;\r\n\r\nDECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)\r\n;WITH\r\ncte0 AS\r\n(\r\n SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1\r\n FROM dbo.Resource\r\n WHERE IsHistory = 0\r\n\t AND IsDeleted = 0\r\n\t AND ResourceTypeId = 40\r\n)\r\n,cte1 AS\r\n(\r\n SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *\r\n FROM\r\n (\r\n\t SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial\r\n\t FROM cte0\r\n\t ORDER BY T1 ASC, Sid1 ASC\r\n ) t\r\n)\r\nINSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1\r\n;WITH cte1 AS (SELECT * FROM @FilteredData)\r\n,cte2 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)\r\n)\r\n,cte3 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte2\r\n)\r\n,cte4 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 404\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte5 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte4\r\n)\r\n,cte6 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte7 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte6\r\n)\r\n,cte8 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte9 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte8\r\n)\r\n,cte10 AS\r\n(\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte1\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte9 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte9.Sid1 AND cte1.T1 = cte9.T1)\r\n)\r\nSELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource\r\nFROM dbo.Resource r\r\n JOIN cte10 ON r.ResourceTypeId = cte10.T1 AND r.ResourceSurrogateId = cte10.Sid1\r\nWHERE IsHistory = 0\r\n AND IsDeleted = 0\r\nORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC\r\n\r\nOPTION (RECOMPILE)\r\n-- execution timeout = 180 sec."; + public SqlCommandSimplifierTests() { _logger = Substitute.For(); @@ -97,5 +103,33 @@ public void GivenACommandThatFailsSimplification_WhenSimplified_ThenNothingIsCha string actualString = stringBuilder.ToString(); Assert.Equal(expectedString, actualString); } + + [Fact] + public void GivenACommandThatContainsIterativeIncludesWithSharedTypes_WhenSimplified_ThenCtesAreUnioned() + { + string startingString = _iterativeIncludeStartingQuery; + string expectedString = _iterativeIncludeEndingQuery; + + IndentedStringBuilder stringBuilder = new IndentedStringBuilder(new StringBuilder(startingString)); + + SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); + + string actualString = stringBuilder.ToString(); + Assert.Equal(expectedString, actualString); + } + + [Fact] + public void GivenACommandThatContainsIterativeIncludesWithoutSharedTypes_WhenSimplified_ThenNothingIsChanged() + { + string startingString = _iterativeIncludeUnchangedQuery; + string expectedString = _iterativeIncludeUnchangedQuery; + + IndentedStringBuilder stringBuilder = new IndentedStringBuilder(new StringBuilder(startingString)); + + SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); + + string actualString = stringBuilder.ToString(); + Assert.Equal(expectedString, actualString); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs index 49a0e0a8f1..50647d1fd7 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs @@ -15,15 +15,15 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Search { internal static class SqlCommandSimplifier { - private static Regex s_findCteMatch = new Regex(",cte(\\d+) AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = (\\d*)\\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\((\\d*)\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte(\\d+) WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"); + private static readonly Regex FindCteMatch = new Regex(",cte(\\d+) AS\\s*\\r\\n\\s*\\(\\s*\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\\s*\\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\s*\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\s*\\r\\n\\s*WHERE refSource.SearchParamId = (\\d*)\\s*\\r\\n\\s*AND refTarget.IsHistory = 0\\s*\\r\\n\\s*AND refTarget.IsDeleted = 0\\s*\\r\\n\\s*AND refSource.ResourceTypeId IN \\((\\d*)\\)\\s*\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte(\\d+) WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"); - private static string s_removeCteMatchBase = "(\\s*,cte AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch \\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\r\\n\\s*WHERE refSource.SearchParamId = \\r\\n\\s*AND refTarget.IsHistory = 0 \\r\\n\\s*AND refTarget.IsDeleted = 0 \\r\\n\\s*AND refSource.ResourceTypeId IN \\(\\)\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1.*\\r\\n\\s*\\)\\r\\n\\s*,cte AS\\r\\n\\s*\\(\\r\\n\\s*SELECT DISTINCT .*T1, Sid1, IsMatch, .* AS IsPartial \\r\\n\\s*FROM cte\\r\\n\\s*\\))"; + private const string RemoveCteMatchBase = "(\\s*,cte AS\\s*\\r\\n\\s*\\(\\s*\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\\s*\\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\s*\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\s*\\r\\n\\s*WHERE refSource.SearchParamId = \\s*\\r\\n\\s*AND refTarget.IsHistory = 0\\s*\\r\\n\\s*AND refTarget.IsDeleted = 0\\s*\\r\\n\\s*AND refSource.ResourceTypeId IN \\(\\)\\s*\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1.*\\r\\n\\s*\\)\\s*\\r\\n\\s*,cte AS\\s*\\r\\n\\s*\\(\\s*\\r\\n\\s*SELECT DISTINCT .*T1, Sid1, IsMatch, .* AS IsPartial\\s*\\r\\n\\s*FROM cte\\s*\\r\\n\\s*\\))"; - private static string s_removeUnionSegmentMatchBase = "(\\s*UNION ALL\\r\\n\\s*SELECT T1, Sid1, IsMatch, IsPartial\\r\\n\\s*FROM cte WHERE NOT EXISTS \\(SELECT \\* FROM cte\\d+ WHERE cte\\d+.Sid1 = cte.Sid1 AND cte\\d+.T1 = cte.T1\\))"; + private const string RemoveUnionSegmentMatchBase = "(\\s*UNION ALL\\s*\\r\\n\\s*SELECT T1, Sid1, IsMatch, IsPartial\\s*\\r\\n\\s*FROM cte WHERE NOT EXISTS \\(SELECT \\* FROM cte\\d+ WHERE cte\\d+.Sid1 = cte.Sid1 AND cte\\d+.T1 = cte.T1\\))"; - private static string s_existsSelectStatement = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; + private const string ExistsSelectStatement = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; - private static string s_unionExistsSelectStatment = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; + private const string UnionExistsSelectStatment = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; internal static void CombineIterativeIncludes(IndentedStringBuilder stringBuilder, ILogger logger) { @@ -82,7 +82,7 @@ internal static void CombineIterativeIncludes(IndentedStringBuilder stringBuilde private static List GetIncludeCteParams(string commandText) { var ctes = new List(); - var cteMatches = s_findCteMatch.Matches(commandText); + var cteMatches = FindCteMatch.Matches(commandText); foreach (Match match in cteMatches) { @@ -100,7 +100,7 @@ private static List GetIncludeCteParams(string comma private static string RemoveCtePair(string commandText, ReferenceSearchInformation cteInfo) { - var removeCteMatch = s_removeCteMatchBase + var removeCteMatch = RemoveCteMatchBase .Replace("", cteInfo.CteNumber.ToString(), StringComparison.Ordinal) .Replace("", cteInfo.SearchParamId.ToString(), StringComparison.Ordinal) .Replace("", cteInfo.ResourceTypeId.ToString(), StringComparison.Ordinal) @@ -120,7 +120,7 @@ private static string RemoveCtePair(string commandText, ReferenceSearchInformati commandText = commandText.Remove(commandText.IndexOf(matches[0].Value, StringComparison.Ordinal), matches[0].Value.Length); - var removeUnionSegmentMatch = s_removeUnionSegmentMatchBase + var removeUnionSegmentMatch = RemoveUnionSegmentMatchBase .Replace("", (cteInfo.CteNumber + 1).ToString(), StringComparison.Ordinal); var removeUnionSegmentRegex = new Regex(removeUnionSegmentMatch); var unionSegmentMatches = removeUnionSegmentRegex.Matches(commandText); @@ -141,7 +141,7 @@ private static string RemoveCtePair(string commandText, ReferenceSearchInformati private static string UnionCtes(string commandText, (ReferenceSearchInformation can1, ReferenceSearchInformation can2) unionCanidate) { - var unionCteMatch = s_removeCteMatchBase + var unionCteMatch = RemoveCteMatchBase .Replace("", unionCanidate.can2.CteNumber.ToString(), StringComparison.Ordinal) .Replace("", unionCanidate.can2.SearchParamId.ToString(), StringComparison.Ordinal) .Replace("", unionCanidate.can2.ResourceTypeId.ToString(), StringComparison.Ordinal) @@ -159,8 +159,8 @@ private static string UnionCtes(string commandText, (ReferenceSearchInformation throw new ArgumentException("No matches found for union cte"); } - var existsSelectStatement = s_existsSelectStatement.Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal); - var newExistsSelectStatement = s_unionExistsSelectStatment + var existsSelectStatement = ExistsSelectStatement.Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal); + var newExistsSelectStatement = UnionExistsSelectStatment .Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal) .Replace("", unionCanidate.can1.SourceCte.ToString(), StringComparison.Ordinal); var newCte = matches[0].Value.Replace(existsSelectStatement, newExistsSelectStatement, StringComparison.Ordinal); From 225777b892069928ee0d502bf3a1e2318e0ac93e Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Thu, 14 Nov 2024 14:30:48 -0800 Subject: [PATCH 4/6] Change simplification method --- .../Search/SqlCommandSimplifierTests.cs | 34 -- .../QueryGenerators/SqlQueryGenerator.cs | 67 ++- .../Features/Search/SqlCommandSimplifier.cs | 166 ------- .../Features/Search/SqlServerSearchService.cs | 406 +++++++++--------- .../Rest/Search/IncludeSearchTests.cs | 6 +- 5 files changed, 227 insertions(+), 452 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs index 46c0a07b0c..2bfd278dd0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlCommandSimplifierTests.cs @@ -21,12 +21,6 @@ public class SqlCommandSimplifierTests { private ILogger _logger; - private readonly string _iterativeIncludeStartingQuery = "DECLARE @p0 int = 11\r\nDECLARE @p1 int = 11\r\nDECLARE @p2 int = 100\r\nDECLARE @p3 int = 100\r\nDECLARE @p4 int = 100\r\nDECLARE @p5 int = 100\r\n\r\nSET STATISTICS IO ON;\r\nSET STATISTICS TIME ON;\r\n\r\nDECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)\r\n;WITH\r\ncte0 AS\r\n(\r\n SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1\r\n FROM dbo.Resource\r\n WHERE IsHistory = 0\r\n\t AND IsDeleted = 0\r\n\t AND ResourceTypeId = 40\r\n)\r\n,cte1 AS\r\n(\r\n SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *\r\n FROM\r\n (\r\n\t SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial\r\n\t FROM cte0\r\n\t ORDER BY T1 ASC, Sid1 ASC\r\n ) t\r\n)\r\nINSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1\r\n;WITH cte1 AS (SELECT * FROM @FilteredData)\r\n,cte2 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)\r\n)\r\n,cte3 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte2\r\n)\r\n,cte4 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 404\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte5 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte4\r\n)\r\n,cte6 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (124)\r\n\t AND EXISTS (SELECT * FROM cte5 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte7 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte6\r\n)\r\n,cte8 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte9 AS\r\n(\r\n SELECT DISTINCT TOP (@p2) T1, Sid1, IsMatch, CASE WHEN count_big(*) over() > @p3 THEN 1 ELSE 0 END AS IsPartial\r\n FROM cte8\r\n)\r\n,cte10 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte11 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte10\r\n)\r\n,cte12 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte9 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte13 AS\r\n(\r\n SELECT DISTINCT TOP (@p4) T1, Sid1, IsMatch, CASE WHEN count_big(*) over() > @p5 THEN 1 ELSE 0 END AS IsPartial\r\n FROM cte12\r\n)\r\n,cte14 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte11 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte15 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte14\r\n)\r\n,cte16 AS\r\n(\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte1\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte9 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte9.Sid1 AND cte1.T1 = cte9.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte11 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte11.Sid1 AND cte1.T1 = cte11.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte13 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte13.Sid1 AND cte1.T1 = cte13.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte15 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte15.Sid1 AND cte1.T1 = cte15.T1)\r\n)\r\nSELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource\r\nFROM dbo.Resource r\r\n JOIN cte16 ON r.ResourceTypeId = cte16.T1 AND r.ResourceSurrogateId = cte16.Sid1\r\nWHERE IsHistory = 0\r\n AND IsDeleted = 0\r\nORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC\r\n\r\nOPTION (RECOMPILE)\r\n-- execution timeout = 180 sec."; - - private readonly string _iterativeIncludeEndingQuery = "DECLARE @p0 int = 11\r\nDECLARE @p1 int = 11\r\nDECLARE @p2 int = 100\r\nDECLARE @p3 int = 100\r\nDECLARE @p4 int = 100\r\nDECLARE @p5 int = 100\r\n\r\nSET STATISTICS IO ON;\r\nSET STATISTICS TIME ON;\r\n\r\nDECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)\r\n;WITH\r\ncte0 AS\r\n(\r\n SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1\r\n FROM dbo.Resource\r\n WHERE IsHistory = 0\r\n\t AND IsDeleted = 0\r\n\t AND ResourceTypeId = 40\r\n)\r\n,cte1 AS\r\n(\r\n SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *\r\n FROM\r\n (\r\n\t SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial\r\n\t FROM cte0\r\n\t ORDER BY T1 ASC, Sid1 ASC\r\n ) t\r\n)\r\nINSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1\r\n;WITH cte1 AS (SELECT * FROM @FilteredData)\r\n,cte2 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)\r\n)\r\n,cte3 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte2\r\n)\r\n,cte4 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 404\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte5 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte4\r\n)\r\n,cte6 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (124)\r\n\t AND EXISTS (SELECT * FROM cte5 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte7 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte6\r\n)\r\n,cte10 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte11 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte10\r\n)\r\n,cte14 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte11 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte15 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte14\r\n)\r\n,cte16 AS\r\n(\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte1\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte11 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte11.Sid1 AND cte1.T1 = cte11.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte15 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte15.Sid1 AND cte1.T1 = cte15.T1)\r\n)\r\nSELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource\r\nFROM dbo.Resource r\r\n JOIN cte16 ON r.ResourceTypeId = cte16.T1 AND r.ResourceSurrogateId = cte16.Sid1\r\nWHERE IsHistory = 0\r\n AND IsDeleted = 0\r\nORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC\r\n\r\nOPTION (RECOMPILE)\r\n-- execution timeout = 180 sec."; - - private readonly string _iterativeIncludeUnchangedQuery = "DECLARE @p0 int = 11\r\nDECLARE @p1 int = 11\r\n\r\nSET STATISTICS IO ON;\r\nSET STATISTICS TIME ON;\r\n\r\nDECLARE @FilteredData AS TABLE (T1 smallint, Sid1 bigint, IsMatch bit, IsPartial bit, Row int)\r\n;WITH\r\ncte0 AS\r\n(\r\n SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1\r\n FROM dbo.Resource\r\n WHERE IsHistory = 0\r\n\t AND IsDeleted = 0\r\n\t AND ResourceTypeId = 40\r\n)\r\n,cte1 AS\r\n(\r\n SELECT row_number() OVER (ORDER BY T1 ASC, Sid1 ASC) AS Row, *\r\n FROM\r\n (\r\n\t SELECT DISTINCT TOP (@p0) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial\r\n\t FROM cte0\r\n\t ORDER BY T1 ASC, Sid1 ASC\r\n ) t\r\n)\r\nINSERT INTO @FilteredData SELECT T1, Sid1, IsMatch, IsPartial, Row FROM cte1\r\n;WITH cte1 AS (SELECT * FROM @FilteredData)\r\n,cte2 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 204\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 AND Row < @p1)\r\n)\r\n,cte3 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte2\r\n)\r\n,cte4 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 404\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (40)\r\n\t AND EXISTS (SELECT * FROM cte1 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte5 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte4\r\n)\r\n,cte6 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 470\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (44)\r\n\t AND EXISTS (SELECT * FROM cte3 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte7 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte6\r\n)\r\n,cte8 AS\r\n(\r\n SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\r\n FROM dbo.ReferenceSearchParam refSource\r\n\t JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\r\n WHERE refSource.SearchParamId = 770\r\n\t AND refTarget.IsHistory = 0\r\n\t AND refTarget.IsDeleted = 0\r\n\t AND refSource.ResourceTypeId IN (71)\r\n\t AND EXISTS (SELECT * FROM cte7 WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1)\r\n)\r\n,cte9 AS\r\n(\r\n SELECT DISTINCT T1, Sid1, IsMatch, 0 AS IsPartial\r\n FROM cte8\r\n)\r\n,cte10 AS\r\n(\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte1\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte3 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte3.Sid1 AND cte1.T1 = cte3.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte5 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte5.Sid1 AND cte1.T1 = cte5.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte7 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte7.Sid1 AND cte1.T1 = cte7.T1)\r\n UNION ALL\r\n SELECT T1, Sid1, IsMatch, IsPartial\r\n FROM cte9 WHERE NOT EXISTS (SELECT * FROM cte1 WHERE cte1.Sid1 = cte9.Sid1 AND cte1.T1 = cte9.T1)\r\n)\r\nSELECT DISTINCT r.ResourceTypeId, r.ResourceId, r.Version, r.IsDeleted, r.ResourceSurrogateId, r.RequestMethod, CAST(IsMatch AS bit) AS IsMatch, CAST(IsPartial AS bit) AS IsPartial, r.IsRawResourceMetaSet, r.SearchParamHash, r.RawResource\r\nFROM dbo.Resource r\r\n JOIN cte10 ON r.ResourceTypeId = cte10.T1 AND r.ResourceSurrogateId = cte10.Sid1\r\nWHERE IsHistory = 0\r\n AND IsDeleted = 0\r\nORDER BY IsMatch DESC, r.ResourceTypeId ASC, r.ResourceSurrogateId ASC\r\n\r\nOPTION (RECOMPILE)\r\n-- execution timeout = 180 sec."; - public SqlCommandSimplifierTests() { _logger = Substitute.For(); @@ -103,33 +97,5 @@ public void GivenACommandThatFailsSimplification_WhenSimplified_ThenNothingIsCha string actualString = stringBuilder.ToString(); Assert.Equal(expectedString, actualString); } - - [Fact] - public void GivenACommandThatContainsIterativeIncludesWithSharedTypes_WhenSimplified_ThenCtesAreUnioned() - { - string startingString = _iterativeIncludeStartingQuery; - string expectedString = _iterativeIncludeEndingQuery; - - IndentedStringBuilder stringBuilder = new IndentedStringBuilder(new StringBuilder(startingString)); - - SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); - - string actualString = stringBuilder.ToString(); - Assert.Equal(expectedString, actualString); - } - - [Fact] - public void GivenACommandThatContainsIterativeIncludesWithoutSharedTypes_WhenSimplified_ThenNothingIsChanged() - { - string startingString = _iterativeIncludeUnchangedQuery; - string expectedString = _iterativeIncludeUnchangedQuery; - - IndentedStringBuilder stringBuilder = new IndentedStringBuilder(new StringBuilder(startingString)); - - SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); - - string actualString = stringBuilder.ToString(); - Assert.Equal(expectedString, actualString); - } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index 126bd55a04..460631e86e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -41,7 +41,6 @@ internal class SqlQueryGenerator : DefaultSqlExpressionVisitor _includeFromCteIds; - private int _curFromCteIndex = -1; private int _tableExpressionCounter = -1; private SqlRootExpression _rootExpression; private readonly SchemaInformation _schemaInfo; @@ -767,6 +766,7 @@ private void HandleTableKindChain( } } + // TODO: Investigate here private void HandleTableKindInclude( SearchParamTableExpression searchParamTableExpression, SearchOptions context, @@ -848,7 +848,9 @@ private void HandleTableKindInclude( .Append(")"); // Get FROM ctes - string fromCte = _cteMainSelect; + List fromCte = new List(); + fromCte.Add(_cteMainSelect); + if (includeExpression.Iterate) { // Include Iterate @@ -858,7 +860,7 @@ private void HandleTableKindInclude( // On that case, the fromCte is _cteMainSelect if (TryGetIncludeCtes(includeExpression.SourceResourceType, out _includeFromCteIds)) { - fromCte = _includeFromCteIds[++_curFromCteIndex]; + fromCte = _includeFromCteIds; } } @@ -869,7 +871,7 @@ private void HandleTableKindInclude( { if (TryGetIncludeCtes(includeExpression.TargetResourceType, out _includeFromCteIds)) { - fromCte = _includeFromCteIds[++_curFromCteIndex]; + fromCte = _includeFromCteIds; } } else if (includeExpression.ReferenceSearchParameter?.TargetResourceTypes != null) @@ -884,7 +886,7 @@ private void HandleTableKindInclude( } _includeFromCteIds = _includeFromCteIds.Distinct().ToList(); - fromCte = _includeFromCteIds.Count > 0 ? _includeFromCteIds[++_curFromCteIndex] : fromCte; + fromCte = _includeFromCteIds.Count > 0 ? _includeFromCteIds : fromCte; } } } @@ -895,19 +897,31 @@ private void HandleTableKindInclude( .Append(" = ").Append(Parameters.AddParameter(VLatest.ReferenceSearchParam.ResourceTypeId, Model.GetResourceTypeId(includeExpression.SourceResourceType), true)); } - delimited.BeginDelimitedElement().Append("EXISTS (SELECT * FROM ").Append(fromCte) - .Append(" WHERE ").Append(VLatest.Resource.ResourceTypeId, table).Append(" = T1 AND ") - .Append(VLatest.Resource.ResourceSurrogateId, table).Append(" = Sid1"); - - if (!includeExpression.Iterate) + // Use _includeFromCteIds and union all tables here, remove section below + var scope = delimited.BeginDelimitedElement(); + scope.Append("EXISTS ("); + for (var index = 0; index < fromCte.Count; index++) { - // Limit the join to the main select CTE. - // The main select will have max+1 items in the result set to account for paging, so we only want to join using the max amount. + var cte = fromCte[index]; + scope.Append("SELECT * FROM ").Append(cte) + .Append(" WHERE ").Append(VLatest.Resource.ResourceTypeId, table).Append(" = T1 AND ") + .Append(VLatest.Resource.ResourceSurrogateId, table).Append(" = Sid1"); - StringBuilder.Append(" AND Row < ").Append(Parameters.AddParameter(context.MaxItemCount + 1, true)); + if (!includeExpression.Iterate) + { + // Limit the join to the main select CTE. + // The main select will have max+1 items in the result set to account for paging, so we only want to join using the max amount. + + scope.Append(" AND Row < ").Append(Parameters.AddParameter(context.MaxItemCount + 1, true)); + } + + if (index < fromCte.Count - 1) + { + scope.AppendLine(" UNION ALL "); + } } - StringBuilder.Append(")"); + scope.Append(")"); } if (includeExpression.Reversed) @@ -941,30 +955,9 @@ private void HandleTableKindInclude( } } - // Handle Multiple Results sets to include from - if (count > 1 && _curFromCteIndex >= 0 && _curFromCteIndex < count - 1) - { - StringBuilder.AppendLine(")").Append(","); - - // If it's not the last result set, append a new IncludeLimit cte, since IncludeLimitCte was not created for the current cte - if (_curFromCteIndex < count - 1) - { - var cteToLimit = TableExpressionName(_tableExpressionCounter); - WriteIncludeLimitCte(cteToLimit, context); - } - - // Generate CTE to include from the additional result sets - StringBuilder.Append(TableExpressionName(++_tableExpressionCounter)).AppendLine(" AS").AppendLine("("); - searchParamTableExpression.AcceptVisitor(this, context); - } - else + if (includeExpression.WildCard) { - _curFromCteIndex = -1; - - if (includeExpression.WildCard) - { - includeExpression.ReferencedTypes?.ToList().ForEach(t => AddIncludeLimitCte(t, curLimitCte)); - } + includeExpression.ReferencedTypes?.ToList().ForEach(t => AddIncludeLimitCte(t, curLimitCte)); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs index 50647d1fd7..eee9bd72a6 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlCommandSimplifier.cs @@ -15,161 +15,6 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Search { internal static class SqlCommandSimplifier { - private static readonly Regex FindCteMatch = new Regex(",cte(\\d+) AS\\s*\\r\\n\\s*\\(\\s*\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\\s*\\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\s*\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\s*\\r\\n\\s*WHERE refSource.SearchParamId = (\\d*)\\s*\\r\\n\\s*AND refTarget.IsHistory = 0\\s*\\r\\n\\s*AND refTarget.IsDeleted = 0\\s*\\r\\n\\s*AND refSource.ResourceTypeId IN \\((\\d*)\\)\\s*\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte(\\d+) WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"); - - private const string RemoveCteMatchBase = "(\\s*,cte AS\\s*\\r\\n\\s*\\(\\s*\\r\\n\\s*SELECT DISTINCT refTarget.ResourceTypeId AS T1, refTarget.ResourceSurrogateId AS Sid1, 0 AS IsMatch\\s*\\r\\n\\s*FROM dbo.ReferenceSearchParam refSource\\s*\\r\\n\\s*JOIN dbo.Resource refTarget ON refSource.ReferenceResourceTypeId = refTarget.ResourceTypeId AND refSource.ReferenceResourceId = refTarget.ResourceId\\s*\\r\\n\\s*WHERE refSource.SearchParamId = \\s*\\r\\n\\s*AND refTarget.IsHistory = 0\\s*\\r\\n\\s*AND refTarget.IsDeleted = 0\\s*\\r\\n\\s*AND refSource.ResourceTypeId IN \\(\\)\\s*\\r\\n\\s*AND EXISTS \\(SELECT \\* FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1.*\\r\\n\\s*\\)\\s*\\r\\n\\s*,cte AS\\s*\\r\\n\\s*\\(\\s*\\r\\n\\s*SELECT DISTINCT .*T1, Sid1, IsMatch, .* AS IsPartial\\s*\\r\\n\\s*FROM cte\\s*\\r\\n\\s*\\))"; - - private const string RemoveUnionSegmentMatchBase = "(\\s*UNION ALL\\s*\\r\\n\\s*SELECT T1, Sid1, IsMatch, IsPartial\\s*\\r\\n\\s*FROM cte WHERE NOT EXISTS \\(SELECT \\* FROM cte\\d+ WHERE cte\\d+.Sid1 = cte.Sid1 AND cte\\d+.T1 = cte.T1\\))"; - - private const string ExistsSelectStatement = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; - - private const string UnionExistsSelectStatment = "SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1 UNION SELECT * FROM cte WHERE refSource.ResourceTypeId = T1 AND refSource.ResourceSurrogateId = Sid1"; - - internal static void CombineIterativeIncludes(IndentedStringBuilder stringBuilder, ILogger logger) - { - var commandText = stringBuilder.ToString(); - try - { - var cteParams = GetIncludeCteParams(commandText); - - var index = 0; - var redundantCtes = new List<(ReferenceSearchInformation, ReferenceSearchInformation)>(); - var unionCanidateCtes = new List<(ReferenceSearchInformation, ReferenceSearchInformation)>(); - foreach (var item in cteParams) - { - for (int i = index + 1; i < cteParams.Count; i++) - { - if (item.SearchParamId == cteParams[i].SearchParamId && item.ResourceTypeId == cteParams[i].ResourceTypeId) - { - if (!unionCanidateCtes.Exists((unionCte) => - (unionCte.Item1.CteNumber + 1 == item.SourceCte && unionCte.Item2.CteNumber + 1 == cteParams[i].SourceCte) - || (unionCte.Item2.CteNumber + 1 == item.SourceCte && unionCte.Item1.CteNumber + 1 == cteParams[i].SourceCte))) - { - unionCanidateCtes.Add((item, cteParams[i])); - } - else - { - redundantCtes.Add((item, cteParams[i])); - } - } - } - - index++; - } - - foreach (var redundantCte in redundantCtes) - { - // Removes the iterate cte and its projection cte, and cleans up the union at the end. - commandText = RemoveCtePair(commandText, redundantCte.Item1); - } - - foreach (var unionCanidate in unionCanidateCtes) - { - // Unions the two ctes together and removes the first one. - commandText = UnionCtes(commandText, unionCanidate); - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Exception combining iterative includes."); - return; // Use the unmodified string - } - - stringBuilder.Clear(); - stringBuilder.Append(commandText); - } - - private static List GetIncludeCteParams(string commandText) - { - var ctes = new List(); - var cteMatches = FindCteMatch.Matches(commandText); - - foreach (Match match in cteMatches) - { - ctes.Add(new ReferenceSearchInformation() - { - SearchParamId = int.Parse(match.Groups[2].Value), - ResourceTypeId = int.Parse(match.Groups[3].Value), - SourceCte = int.Parse(match.Groups[4].Value), - CteNumber = int.Parse(match.Groups[1].Value), - }); - } - - return ctes; - } - - private static string RemoveCtePair(string commandText, ReferenceSearchInformation cteInfo) - { - var removeCteMatch = RemoveCteMatchBase - .Replace("", cteInfo.CteNumber.ToString(), StringComparison.Ordinal) - .Replace("", cteInfo.SearchParamId.ToString(), StringComparison.Ordinal) - .Replace("", cteInfo.ResourceTypeId.ToString(), StringComparison.Ordinal) - .Replace("", cteInfo.SourceCte.ToString(), StringComparison.Ordinal) - .Replace("", (cteInfo.CteNumber + 1).ToString(), StringComparison.Ordinal); - var removeCteRegex = new Regex(removeCteMatch); - var matches = removeCteRegex.Matches(commandText); - - if (matches.Count > 1) - { - throw new ArgumentException("More than one match found for cte to remove"); - } - else if (matches.Count == 0) - { - throw new ArgumentException("No matches found for cte to remove"); - } - - commandText = commandText.Remove(commandText.IndexOf(matches[0].Value, StringComparison.Ordinal), matches[0].Value.Length); - - var removeUnionSegmentMatch = RemoveUnionSegmentMatchBase - .Replace("", (cteInfo.CteNumber + 1).ToString(), StringComparison.Ordinal); - var removeUnionSegmentRegex = new Regex(removeUnionSegmentMatch); - var unionSegmentMatches = removeUnionSegmentRegex.Matches(commandText); - - if (unionSegmentMatches.Count > 1) - { - throw new ArgumentException("More than one match found for union segment to remove"); - } - else if (unionSegmentMatches.Count == 0) - { - throw new ArgumentException("No matches found for union segment to remove"); - } - - commandText = commandText.Remove(commandText.IndexOf(unionSegmentMatches[0].Value, StringComparison.Ordinal), unionSegmentMatches[0].Value.Length); - - return commandText; - } - - private static string UnionCtes(string commandText, (ReferenceSearchInformation can1, ReferenceSearchInformation can2) unionCanidate) - { - var unionCteMatch = RemoveCteMatchBase - .Replace("", unionCanidate.can2.CteNumber.ToString(), StringComparison.Ordinal) - .Replace("", unionCanidate.can2.SearchParamId.ToString(), StringComparison.Ordinal) - .Replace("", unionCanidate.can2.ResourceTypeId.ToString(), StringComparison.Ordinal) - .Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal) - .Replace("", (unionCanidate.can2.CteNumber + 1).ToString(), StringComparison.Ordinal); - var unionCteRegex = new Regex(unionCteMatch); - var matches = unionCteRegex.Matches(commandText); - - if (matches.Count > 1) - { - throw new ArgumentException("More than one match found for union cte"); - } - else if (matches.Count == 0) - { - throw new ArgumentException("No matches found for union cte"); - } - - var existsSelectStatement = ExistsSelectStatement.Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal); - var newExistsSelectStatement = UnionExistsSelectStatment - .Replace("", unionCanidate.can2.SourceCte.ToString(), StringComparison.Ordinal) - .Replace("", unionCanidate.can1.SourceCte.ToString(), StringComparison.Ordinal); - var newCte = matches[0].Value.Replace(existsSelectStatement, newExistsSelectStatement, StringComparison.Ordinal); - commandText = commandText.Replace(matches[0].Value, newCte, StringComparison.Ordinal); - - commandText = RemoveCtePair(commandText, unionCanidate.can1); - return commandText; - } - internal static void RemoveRedundantParameters(IndentedStringBuilder stringBuilder, SqlParameterCollection sqlParameterCollection, ILogger logger) { var commandText = stringBuilder.ToString(); @@ -250,16 +95,5 @@ private static string RemoveRedundantComparisons(string commandText, SqlParamete return commandText; } - - private class ReferenceSearchInformation - { - public int CteNumber { get; set; } - - public int SearchParamId { get; set; } - - public int ResourceTypeId { get; set; } - - public int SourceCte { get; set; } - } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index a472422e7f..e8c33eb108 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -224,7 +224,7 @@ public override async Task SearchAsync(SearchOptions searchOptions return searchResult; } - private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, CancellationToken cancellationToken, bool skipSimplification = false) + private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, CancellationToken cancellationToken) { Expression searchExpression = sqlSearchOptions.Expression; @@ -332,266 +332,246 @@ private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, C await CreateStats(expression, cancellationToken); SearchResult searchResult = null; - var commandSimplified = false; - try - { - await _sqlRetryService.ExecuteSql( - async (connection, cancellationToken, sqlException) => + await _sqlRetryService.ExecuteSql( + async (connection, cancellationToken, sqlException) => + { + using (SqlCommand sqlCommand = connection.CreateCommand()) // WARNING, this code will not set sqlCommand.Transaction. Sql transactions via C#/.NET are not supported in this method. { - using (SqlCommand sqlCommand = connection.CreateCommand()) // WARNING, this code will not set sqlCommand.Transaction. Sql transactions via C#/.NET are not supported in this method. + sqlCommand.CommandTimeout = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; + + var exportTimeTravel = clonedSearchOptions.QueryHints != null && ContainsGlobalEndSurrogateId(clonedSearchOptions); + if (exportTimeTravel) + { + PopulateSqlCommandFromQueryHints(clonedSearchOptions, sqlCommand); + sqlCommand.CommandTimeout = 1200; // set to 20 minutes, as dataset is usually large + } + else { - sqlCommand.CommandTimeout = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; + var stringBuilder = new IndentedStringBuilder(new StringBuilder()); - var exportTimeTravel = clonedSearchOptions.QueryHints != null && ContainsGlobalEndSurrogateId(clonedSearchOptions); - if (exportTimeTravel) - { - PopulateSqlCommandFromQueryHints(clonedSearchOptions, sqlCommand); - sqlCommand.CommandTimeout = 1200; // set to 20 minutes, as dataset is usually large - } - else - { - var stringBuilder = new IndentedStringBuilder(new StringBuilder()); + EnableTimeAndIoMessageLogging(stringBuilder, connection); - EnableTimeAndIoMessageLogging(stringBuilder, connection); + var queryGenerator = new SqlQueryGenerator( + stringBuilder, + new HashingSqlQueryParameterManager(new SqlQueryParameterManager(sqlCommand.Parameters)), + _model, + _schemaInformation, + _reuseQueryPlans.IsEnabled(_sqlRetryService), + sqlException); - var queryGenerator = new SqlQueryGenerator( - stringBuilder, - new HashingSqlQueryParameterManager(new SqlQueryParameterManager(sqlCommand.Parameters)), - _model, - _schemaInformation, - _reuseQueryPlans.IsEnabled(_sqlRetryService), - sqlException); + expression.AcceptVisitor(queryGenerator, clonedSearchOptions); - expression.AcceptVisitor(queryGenerator, clonedSearchOptions); + SqlCommandSimplifier.RemoveRedundantParameters(stringBuilder, sqlCommand.Parameters, _logger); - if (!skipSimplification) - { - var originonalCommandText = stringBuilder.ToString(); - SqlCommandSimplifier.RemoveRedundantParameters(stringBuilder, sqlCommand.Parameters, _logger); - SqlCommandSimplifier.CombineIterativeIncludes(stringBuilder, _logger); - commandSimplified = originonalCommandText != stringBuilder.ToString(); - } - - var queryText = stringBuilder.ToString(); - var queryHash = _queryHashCalculator.CalculateHash(queryText); - _logger.LogInformation("SQL Search Service query hash: {QueryHash}", queryHash); - var customQuery = CustomQueries.CheckQueryHash(connection, queryHash, _logger); + var queryText = stringBuilder.ToString(); + var queryHash = _queryHashCalculator.CalculateHash(queryText); + _logger.LogInformation("SQL Search Service query hash: {QueryHash}", queryHash); + var customQuery = CustomQueries.CheckQueryHash(connection, queryHash, _logger); - if (!string.IsNullOrEmpty(customQuery)) - { - _logger.LogInformation("SQl Search Service, custom Query identified by hash {QueryHash}, {CustomQuery}", queryHash, customQuery); - queryText = customQuery; - sqlCommand.CommandType = CommandType.StoredProcedure; - } + if (!string.IsNullOrEmpty(customQuery)) + { + _logger.LogInformation("SQl Search Service, custom Query identified by hash {QueryHash}, {CustomQuery}", queryHash, customQuery); + queryText = customQuery; + sqlCommand.CommandType = CommandType.StoredProcedure; + } - // Command text contains no direct user input. + // Command text contains no direct user input. #pragma warning disable CA2100 // Review SQL queries for security vulnerabilities - sqlCommand.CommandText = queryText; + sqlCommand.CommandText = queryText; #pragma warning restore CA2100 // Review SQL queries for security vulnerabilities - } + } - LogSqlCommand(sqlCommand); + LogSqlCommand(sqlCommand); - using (var reader = await sqlCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken)) + using (var reader = await sqlCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken)) + { + if (clonedSearchOptions.CountOnly) { - if (clonedSearchOptions.CountOnly) + await reader.ReadAsync(cancellationToken); + long count = reader.GetInt64(0); + if (count > int.MaxValue) { - await reader.ReadAsync(cancellationToken); - long count = reader.GetInt64(0); - if (count > int.MaxValue) - { - _requestContextAccessor.RequestContext.BundleIssues.Add( - new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.NotSupported, - string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue))); - - _logger.LogWarning("Invalid Search Operation (SearchCountResultsExceedLimit)"); - throw new InvalidSearchOperationException(string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); - } + _requestContextAccessor.RequestContext.BundleIssues.Add( + new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.NotSupported, + string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue))); - searchResult = new SearchResult((int)count, clonedSearchOptions.UnsupportedSearchParams); + _logger.LogWarning("Invalid Search Operation (SearchCountResultsExceedLimit)"); + throw new InvalidSearchOperationException(string.Format(Core.Resources.SearchCountResultsExceedLimit, count, int.MaxValue)); + } - // call NextResultAsync to get the info messages - await reader.NextResultAsync(cancellationToken); + searchResult = new SearchResult((int)count, clonedSearchOptions.UnsupportedSearchParams); - return; - } + // call NextResultAsync to get the info messages + await reader.NextResultAsync(cancellationToken); - var resources = new List(sqlSearchOptions.MaxItemCount); - short? newContinuationType = null; - long? newContinuationId = null; - bool moreResults = false; - int matchCount = 0; + return; + } + + var resources = new List(sqlSearchOptions.MaxItemCount); + short? newContinuationType = null; + long? newContinuationId = null; + bool moreResults = false; + int matchCount = 0; - string sortValue = null; - var isResultPartial = false; - int numberOfColumnsRead = 0; + string sortValue = null; + var isResultPartial = false; + int numberOfColumnsRead = 0; - while (await reader.ReadAsync(cancellationToken)) + while (await reader.ReadAsync(cancellationToken)) + { + ReadWrapper( + reader, + out short resourceTypeId, + out string resourceId, + out int version, + out bool isDeleted, + out long resourceSurrogateId, + out string requestMethod, + out bool isMatch, + out bool isPartialEntry, + out bool isRawResourceMetaSet, + out string searchParameterHash, + out byte[] rawResourceBytes, + out bool isInvisible); + + if (isInvisible) { - ReadWrapper( - reader, - out short resourceTypeId, - out string resourceId, - out int version, - out bool isDeleted, - out long resourceSurrogateId, - out string requestMethod, - out bool isMatch, - out bool isPartialEntry, - out bool isRawResourceMetaSet, - out string searchParameterHash, - out byte[] rawResourceBytes, - out bool isInvisible); - - if (isInvisible) - { - continue; - } + continue; + } - numberOfColumnsRead = reader.FieldCount; + numberOfColumnsRead = reader.FieldCount; - // If we get to this point, we know there are more results so we need a continuation token - // Additionally, this resource shouldn't be included in the results - if (matchCount >= clonedSearchOptions.MaxItemCount && isMatch) - { - moreResults = true; + // If we get to this point, we know there are more results so we need a continuation token + // Additionally, this resource shouldn't be included in the results + if (matchCount >= clonedSearchOptions.MaxItemCount && isMatch) + { + moreResults = true; - continue; - } + continue; + } - Lazy rawResource = new Lazy(() => string.Empty); + Lazy rawResource = new Lazy(() => string.Empty); - if (!clonedSearchOptions.OnlyIds) + if (!clonedSearchOptions.OnlyIds) + { + rawResource = new Lazy(() => { - rawResource = new Lazy(() => - { - using var rawResourceStream = new MemoryStream(rawResourceBytes); - var decompressedResource = _compressedRawResourceConverter.ReadCompressedRawResource(rawResourceStream); + using var rawResourceStream = new MemoryStream(rawResourceBytes); + var decompressedResource = _compressedRawResourceConverter.ReadCompressedRawResource(rawResourceStream); - _logger.LogVerbose(_parameterStore, cancellationToken, "{NameOfResourceSurrogateId}: {ResourceSurrogateId}; {NameOfResourceTypeId}: {ResourceTypeId}; Decompressed length: {RawResourceLength}", nameof(resourceSurrogateId), resourceSurrogateId, nameof(resourceTypeId), resourceTypeId, decompressedResource.Length); + _logger.LogVerbose(_parameterStore, cancellationToken, "{NameOfResourceSurrogateId}: {ResourceSurrogateId}; {NameOfResourceTypeId}: {ResourceTypeId}; Decompressed length: {RawResourceLength}", nameof(resourceSurrogateId), resourceSurrogateId, nameof(resourceTypeId), resourceTypeId, decompressedResource.Length); - if (string.IsNullOrEmpty(decompressedResource)) - { - decompressedResource = MissingResourceFactory.CreateJson(resourceId, _model.GetResourceTypeName(resourceTypeId), "warning", "incomplete"); - _requestContextAccessor.SetMissingResourceCode(System.Net.HttpStatusCode.PartialContent); - } + if (string.IsNullOrEmpty(decompressedResource)) + { + decompressedResource = MissingResourceFactory.CreateJson(resourceId, _model.GetResourceTypeName(resourceTypeId), "warning", "incomplete"); + _requestContextAccessor.SetMissingResourceCode(System.Net.HttpStatusCode.PartialContent); + } - return decompressedResource; - }); - } + return decompressedResource; + }); + } - // See if this resource is a continuation token candidate and increase the count - if (isMatch) - { - newContinuationType = resourceTypeId; - newContinuationId = resourceSurrogateId; + // See if this resource is a continuation token candidate and increase the count + if (isMatch) + { + newContinuationType = resourceTypeId; + newContinuationId = resourceSurrogateId; - // For normal queries, we select _defaultNumberOfColumnsReadFromResult number of columns. - // If we have more, that means we have an extra column tracking sort value. - // Keep track of sort value if this is the last row. - if (matchCount == clonedSearchOptions.MaxItemCount - 1 && reader.FieldCount > _defaultNumberOfColumnsReadFromResult) + // For normal queries, we select _defaultNumberOfColumnsReadFromResult number of columns. + // If we have more, that means we have an extra column tracking sort value. + // Keep track of sort value if this is the last row. + if (matchCount == clonedSearchOptions.MaxItemCount - 1 && reader.FieldCount > _defaultNumberOfColumnsReadFromResult) + { + var tempSortValue = reader.GetValue(SortValueColumnName); + if ((tempSortValue as DateTime?) != null) { - var tempSortValue = reader.GetValue(SortValueColumnName); - if ((tempSortValue as DateTime?) != null) - { - sortValue = (tempSortValue as DateTime?).Value.ToString("o"); - } - else - { - sortValue = tempSortValue.ToString(); - } + sortValue = (tempSortValue as DateTime?).Value.ToString("o"); + } + else + { + sortValue = tempSortValue.ToString(); } - - matchCount++; } - // as long as at least one entry was marked as partial, this resultset - // should be marked as partial - isResultPartial = isResultPartial || isPartialEntry; - - resources.Add(new SearchResultEntry( - new ResourceWrapper( - resourceId, - version.ToString(CultureInfo.InvariantCulture), - _model.GetResourceTypeName(resourceTypeId), - clonedSearchOptions.OnlyIds ? null : new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet), - new ResourceRequest(requestMethod), - resourceSurrogateId.ToLastUpdated(), - isDeleted, - null, - null, - null, - searchParameterHash, - resourceSurrogateId), - isMatch ? SearchEntryMode.Match : SearchEntryMode.Include)); + matchCount++; } - // call NextResultAsync to get the info messages - await reader.NextResultAsync(cancellationToken); + // as long as at least one entry was marked as partial, this resultset + // should be marked as partial + isResultPartial = isResultPartial || isPartialEntry; + + resources.Add(new SearchResultEntry( + new ResourceWrapper( + resourceId, + version.ToString(CultureInfo.InvariantCulture), + _model.GetResourceTypeName(resourceTypeId), + clonedSearchOptions.OnlyIds ? null : new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet), + new ResourceRequest(requestMethod), + resourceSurrogateId.ToLastUpdated(), + isDeleted, + null, + null, + null, + searchParameterHash, + resourceSurrogateId), + isMatch ? SearchEntryMode.Match : SearchEntryMode.Include)); + } - ContinuationToken continuationToken = moreResults - ? new ContinuationToken( - clonedSearchOptions.Sort.Select(s => - s.searchParameterInfo.Name switch - { - SearchParameterNames.ResourceType => (object)newContinuationType, - SearchParameterNames.LastUpdated => newContinuationId, - _ => sortValue, - }).ToArray()) - : null; + // call NextResultAsync to get the info messages + await reader.NextResultAsync(cancellationToken); - if (isResultPartial) - { - _logger.LogWarning("Bundle Partial Result (TruncatedIncludeMessage)"); - _requestContextAccessor.RequestContext.BundleIssues.Add( - new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Warning, - OperationOutcomeConstants.IssueType.Incomplete, - Core.Resources.TruncatedIncludeMessage)); - } + ContinuationToken continuationToken = moreResults + ? new ContinuationToken( + clonedSearchOptions.Sort.Select(s => + s.searchParameterInfo.Name switch + { + SearchParameterNames.ResourceType => (object)newContinuationType, + SearchParameterNames.LastUpdated => newContinuationId, + _ => sortValue, + }).ToArray()) + : null; - // If this is a sort query, lets keep track of whether we actually searched for sort values. - if (clonedSearchOptions.Sort != null && - clonedSearchOptions.Sort.Count > 0 && - clonedSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated) - { - // If there is an extra column for sort value, we know we have searched for sort values. If no results were returned, we don't know if we have searched for sort values so we need to assume we did so we run the second phase. - sqlSearchOptions.DidWeSearchForSortValue = numberOfColumnsRead > _defaultNumberOfColumnsReadFromResult; - } + if (isResultPartial) + { + _logger.LogWarning("Bundle Partial Result (TruncatedIncludeMessage)"); + _requestContextAccessor.RequestContext.BundleIssues.Add( + new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Warning, + OperationOutcomeConstants.IssueType.Incomplete, + Core.Resources.TruncatedIncludeMessage)); + } - // This value is set inside the SortRewriter. If it is set, we need to pass - // this value back to the caller. - if (clonedSearchOptions.IsSortWithFilter) - { - sqlSearchOptions.IsSortWithFilter = true; - } + // If this is a sort query, lets keep track of whether we actually searched for sort values. + if (clonedSearchOptions.Sort != null && + clonedSearchOptions.Sort.Count > 0 && + clonedSearchOptions.Sort[0].searchParameterInfo.Code != KnownQueryParameterNames.LastUpdated) + { + // If there is an extra column for sort value, we know we have searched for sort values. If no results were returned, we don't know if we have searched for sort values so we need to assume we did so we run the second phase. + sqlSearchOptions.DidWeSearchForSortValue = numberOfColumnsRead > _defaultNumberOfColumnsReadFromResult; + } - if (clonedSearchOptions.SortHasMissingModifier) - { - sqlSearchOptions.SortHasMissingModifier = true; - } + // This value is set inside the SortRewriter. If it is set, we need to pass + // this value back to the caller. + if (clonedSearchOptions.IsSortWithFilter) + { + sqlSearchOptions.IsSortWithFilter = true; + } - searchResult = new SearchResult(resources, continuationToken?.ToJson(), originalSort, clonedSearchOptions.UnsupportedSearchParams); + if (clonedSearchOptions.SortHasMissingModifier) + { + sqlSearchOptions.SortHasMissingModifier = true; } - } - }, - _logger, - cancellationToken, - true); // this enables reads from replicas - } - catch (SqlException ex) - { - if (commandSimplified) - { - _logger.LogWarning(ex, "Error executing simplified command. Retrying with original command."); - return await SearchImpl(sqlSearchOptions, cancellationToken, true); - } - throw; - } + searchResult = new SearchResult(resources, continuationToken?.ToJson(), originalSort, clonedSearchOptions.UnsupportedSearchParams); + } + } + }, + _logger, + cancellationToken, + true); // this enables reads from replicas return searchResult; } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTests.cs index af7e1bf92d..895399f91a 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/IncludeSearchTests.cs @@ -671,7 +671,7 @@ await SearchAndValidateBundleAsync( public async Task GivenAnIncludeIterateSearchExpressionWithMultitypeArrayReference_WhenSearched_TheIterativeResultsShouldBeAddedToTheBundle() { // Non-recursive iteration - Reference array of multiple target types: CareTeam:participant of type Patient, Practitioner, Organization, etc. - string query = $"_include=CareTeam:participant:Patient&_include:iterate=Patient:general-practitioner&_tag={Fixture.Tag}"; + string query = $"_include=CareTeam:participant&_include:iterate=Patient:general-practitioner&_tag={Fixture.Tag}"; await SearchAndValidateBundleAsync( ResourceType.CareTeam, @@ -682,7 +682,9 @@ await SearchAndValidateBundleAsync( Fixture.TrumanPatient, Fixture.AndersonPractitioner, Fixture.SanchezPractitioner, - Fixture.TaylorPractitioner); + Fixture.TaylorPractitioner, + Fixture.Organization, + Fixture.Practitioner); } [Fact] From 9c68393e04a51cb7d47560ef216e8d3ded54f827 Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Thu, 14 Nov 2024 14:33:54 -0800 Subject: [PATCH 5/6] Remove comments --- .../Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index 460631e86e..015bb2e5cb 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -766,7 +766,6 @@ private void HandleTableKindChain( } } - // TODO: Investigate here private void HandleTableKindInclude( SearchParamTableExpression searchParamTableExpression, SearchOptions context, @@ -897,7 +896,6 @@ private void HandleTableKindInclude( .Append(" = ").Append(Parameters.AddParameter(VLatest.ReferenceSearchParam.ResourceTypeId, Model.GetResourceTypeId(includeExpression.SourceResourceType), true)); } - // Use _includeFromCteIds and union all tables here, remove section below var scope = delimited.BeginDelimitedElement(); scope.Append("EXISTS ("); for (var index = 0; index < fromCte.Count; index++) From 2ec303a07c797cf0ba965f1639d1dd929eb71331 Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Tue, 19 Nov 2024 07:46:54 -0800 Subject: [PATCH 6/6] Remove unused method --- .../QueryGenerators/SqlQueryGenerator.cs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index 015bb2e5cb..59fd7a84c4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -1132,27 +1132,6 @@ private void HandleTableKindSortWithFilter(SearchParamTableExpression searchPara _sortVisited = true; } - private void WriteIncludeLimitCte(string cteToLimit, SearchOptions context) - { - StringBuilder.Append(TableExpressionName(++_tableExpressionCounter)).AppendLine(" AS").AppendLine("("); - - // the related cte is a reverse include, limit the number of returned items and count to - // see if we are over the threshold (to produce a warning to the client) - StringBuilder.Append("SELECT DISTINCT "); - StringBuilder.Append("TOP (").Append(Parameters.AddParameter(context.IncludeCount, true)).Append(") "); - - StringBuilder.Append("T1, Sid1, IsMatch, "); - StringBuilder.Append("CASE WHEN count_big(*) over() > ") - .Append(Parameters.AddParameter(context.IncludeCount, true)) - .AppendLine(" THEN 1 ELSE 0 END AS IsPartial "); - - StringBuilder.Append("FROM ").AppendLine(cteToLimit); - StringBuilder.AppendLine(")").Append(","); - - // the 'original' include cte is not in the union, but this new layer is instead - _includeCteIds.Add(TableExpressionName(_tableExpressionCounter)); - } - private SearchParameterQueryGeneratorContext GetContext(string tableAlias = null) { return new SearchParameterQueryGeneratorContext(StringBuilder, Parameters, Model, _schemaInfo, tableAlias);