fix(meta) Change the view to use LEFT ARRAY JOIN #5975
Merged
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Change the table validation to also handle the LEFT ARRAY JOIN clause.
In the current implementation if a metric has no tags associated with it, it will get excluded from the meta table since the array join only selects rows that have values. This change will cause rows to be created with an empty tag key (0) so that the metrics can still be searched for in the meta tables.
|
This PR has a migration; here is the generated SQL -- start migrations
-- forward migration generic_metrics : 0048_counters_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_counters_meta_mv;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_counters_meta_mv_v2 TO generic_metric_counters_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_counters_raw_local
LEFT ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end forward migration generic_metrics : 0048_counters_meta_tables_support_empty_tags
-- backward migration generic_metrics : 0048_counters_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_counters_meta_mv_v2;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_counters_meta_mv TO generic_metric_counters_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_counters_raw_local
ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end backward migration generic_metrics : 0048_counters_meta_tables_support_empty_tags
-- forward migration generic_metrics : 0049_sets_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_sets_meta_mv;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_sets_meta_mv_v2 TO generic_metric_sets_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_sets_raw_local
LEFT ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end forward migration generic_metrics : 0049_sets_meta_tables_support_empty_tags
-- backward migration generic_metrics : 0049_sets_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_sets_meta_mv_v2;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_sets_meta_mv TO generic_metric_sets_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_sets_raw_local
ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end backward migration generic_metrics : 0049_sets_meta_tables_support_empty_tags
-- forward migration generic_metrics : 0050_distributions_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_distributions_meta_mv;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_distributions_meta_mv_v2 TO generic_metric_distributions_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_distributions_raw_local
LEFT ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end forward migration generic_metrics : 0050_distributions_meta_tables_support_empty_tags
-- backward migration generic_metrics : 0050_distributions_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_distributions_meta_mv_v2;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_distributions_meta_mv TO generic_metric_distributions_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_distributions_raw_local
ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end backward migration generic_metrics : 0050_distributions_meta_tables_support_empty_tags
-- forward migration generic_metrics : 0051_gauges_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_gauges_meta_mv;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_gauges_meta_mv_v2 TO generic_metric_gauges_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_gauges_raw_local
LEFT ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end forward migration generic_metrics : 0051_gauges_meta_tables_support_empty_tags
-- backward migration generic_metrics : 0051_gauges_meta_tables_support_empty_tags
Local op: DROP TABLE IF EXISTS generic_metric_gauges_meta_mv_v2;
Local op: CREATE MATERIALIZED VIEW IF NOT EXISTS generic_metric_gauges_meta_mv TO generic_metric_gauges_meta_local (org_id UInt64, project_id UInt64, use_case_id LowCardinality(String), metric_id UInt64, tag_key UInt64, timestamp DateTime CODEC (DoubleDelta), retention_days UInt16, count AggregateFunction(sum, Float64)) AS
SELECT
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
toMonday(timestamp) as timestamp,
retention_days,
sumState(count_value) as count
FROM generic_metric_gauges_raw_local
ARRAY JOIN tags.key AS tag_key
WHERE record_meta = 1
GROUP BY
org_id,
project_id,
use_case_id,
metric_id,
tag_key,
timestamp,
retention_days
;
-- end backward migration generic_metrics : 0051_gauges_meta_tables_support_empty_tags |
|
Validated these queries in snuba admin and they work as expected. There might need to be a follow up PR in the metrics layer to handle the |
john-z-yang
approved these changes
May 30, 2024
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
In the current implementation if a metric has no tags associated with it, it will get excluded from the meta table since the array join only selects rows that have values.
This change will cause rows to be created with an empty tag key (0) so that the metrics can still be searched for in the meta tables.
TODO: This depends on #5974 because I still need to validate that thisactually works correctly before this can be merged.