From 5ec512292093fcf81f3edd6c3d7359b6f336e242 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Mon, 3 Jun 2024 15:45:56 +0530 Subject: [PATCH 01/11] fix total.bytes for tablespace --- CHANGELOG.next.asciidoc | 1 + .../05acae50-9bf0-11e9-a61b-f742ed613c57.json | 24 ++++++++++++++++++- .../module/oracle/tablespace/data.go | 3 ++- .../module/oracle/tablespace/mocks_test.go | 10 ++++---- .../oracle/tablespace/temp_free_space.go | 13 +++++----- .../oracle/tablespace/used_and_free_space.go | 12 ++++++---- 6 files changed, 45 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ef638899274..4c4bf1b0a1e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -176,6 +176,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627] - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710] - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664] +:qwq- Fixed query logic for temp and non-temp tablespaces in Oracle module {issue}1[1] {pull}1[1] *Osquerybeat* diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json index 603fcc02ec2..cc48609a438 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json @@ -28,9 +28,31 @@ "line_width": 1, "metrics": [ { - "field": "oracle.tablespace.space.total.bytes", + "field": "oracle.tablespace.space.used.bytes", "id": "61ca57f2-469d-11e7-af02-69e470af7417", "type": "avg" + }, + { + "field": "oracle.tablespace.space.free.bytes", + "id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "type": "avg" + }, + { + "id": "2cf57800-8b54-41fa-a877-159b49699a50", + "script": "params.used_bytes + params.free_bytes", + "type": "math", + "variables": [ + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "631a44d5-d18a-4743-bea0-6f61930fd65f", + "name": "used_bytes" + }, + { + "field": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", + "id": "c255d24c-3a29-4879-b999-77af43d97c6b", + "name": "free_bytes" + } + ] } ], "point_size": 1, diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index 30c86d766da..1baa6252fc3 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -78,7 +78,7 @@ func (m *MetricSet) addTempFreeSpaceData(tempFreeSpaces []tempFreeSpace, out map name := val.(string) if name == "TEMP" { for _, tempFreeSpaceTable := range tempFreeSpaces { - oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TablespaceSize}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TotalSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.UsedSpaceBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.FreeSpace}) } @@ -101,6 +101,7 @@ func (m *MetricSet) addUsedAndFreeSpaceData(freeSpaces []usedAndFreeSpace, out m if name == freeSpaceTable.TablespaceName { oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalFreeBytes}) oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalUsedBytes}) + oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalSpaceBytes}) } } } diff --git a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go index 12348236bcc..3a13dbed352 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go @@ -77,16 +77,16 @@ func (h happyDataFiles) dataFilesData(_ context.Context) ([]dataFile, error) { type happyTempFreeSpaceData struct{} func (happyTempFreeSpaceData) tempFreeSpaceData(_ context.Context) ([]tempFreeSpace, error) { - return []tempFreeSpace{{TablespaceName: "TEMP", TablespaceSize: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil + return []tempFreeSpace{{TablespaceName: "TEMP", TotalSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil } type happyFreeSpaceData struct{} func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) { return []usedAndFreeSpace{ - {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, }, nil } diff --git a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go index efc8401e2c0..94387650e3a 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go @@ -11,10 +11,10 @@ import ( ) type tempFreeSpace struct { - TablespaceName string - TablespaceSize sql.NullInt64 - UsedSpaceBytes sql.NullInt64 - FreeSpace sql.NullInt64 + TablespaceName string + TotalSpaceBytes sql.NullInt64 + UsedSpaceBytes sql.NullInt64 + FreeSpace sql.NullInt64 } func (d *tempFreeSpace) hash() string { @@ -26,16 +26,17 @@ func (d *tempFreeSpace) eventKey() string { } func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT TABLESPACE_NAME, TABLESPACE_SIZE, ALLOCATED_SPACE, FREE_SPACE FROM DBA_TEMP_FREE_SPACE") + rows, err := e.db.QueryContext(ctx, `WITH sums AS ( SELECT (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM FROM dual ) SELECT t.TABLESPACE_NAME, s.TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t, sums s `) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]tempFreeSpace, 0) for rows.Next() { dest := tempFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TablespaceSize, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalSpaceBytes, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil { return nil, err } results = append(results, dest) diff --git a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go index b17b249808d..7e184eafef7 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go @@ -11,9 +11,10 @@ import ( ) type usedAndFreeSpace struct { - TablespaceName string - TotalFreeBytes sql.NullInt64 - TotalUsedBytes sql.NullInt64 + TablespaceName string + TotalSpaceBytes sql.NullInt64 + TotalFreeBytes sql.NullInt64 + TotalUsedBytes sql.NullInt64 } func (d *usedAndFreeSpace) hash() string { @@ -25,16 +26,17 @@ func (d *usedAndFreeSpace) eventKey() string { } func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, "SELECT b.tablespace_name, tbs_size used, a.free_space free FROM (SELECT tablespace_name, sum(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, sum(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+)=b.tablespace_name") + rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - a.free_space) used, a.free_space free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } + defer rows.Close() results := make([]usedAndFreeSpace, 0) for rows.Next() { dest := usedAndFreeSpace{} - if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes); err != nil { + if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes, &dest.TotalSpaceBytes); err != nil { return nil, err } results = append(results, dest) From 4395915538b934c74aa92925f7e349408452b017 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Mon, 3 Jun 2024 16:06:29 +0530 Subject: [PATCH 02/11] update changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4c4bf1b0a1e..7d061c81965 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -176,7 +176,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627] - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710] - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664] -:qwq- Fixed query logic for temp and non-temp tablespaces in Oracle module {issue}1[1] {pull}1[1] +- Fixed query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] *Osquerybeat* From 2e7a99819f601a12a2e6ff6b97bffed07eef0ad3 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Tue, 4 Jun 2024 15:35:14 +0530 Subject: [PATCH 03/11] fix CI errors --- x-pack/metricbeat/module/oracle/tablespace/data.go | 7 ++++--- .../module/oracle/tablespace/temp_free_space.go | 8 -------- .../module/oracle/tablespace/used_and_free_space.go | 8 -------- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index 1baa6252fc3..ae71b3cc766 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -31,7 +31,7 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth return nil, fmt.Errorf("error getting free space data: %w", err) } - return + return out, nil } // transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data @@ -41,13 +41,14 @@ func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { out = make(map[string]mapstr.M, 0) for _, dataFile := range in.dataFiles { - m.addDataFileData(&dataFile, out) + dataFileCopy := dataFile + m.addDataFileData(&dataFileCopy, out) } m.addUsedAndFreeSpaceData(in.freeSpace, out) m.addTempFreeSpaceData(in.tempFreeSpace, out) - return + return out } func (m *MetricSet) extractAndTransform(ctx context.Context) ([]mb.Event, error) { diff --git a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go index 94387650e3a..edf5d59be88 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go @@ -17,14 +17,6 @@ type tempFreeSpace struct { FreeSpace sql.NullInt64 } -func (d *tempFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *tempFreeSpace) eventKey() string { - return d.TablespaceName -} - func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) { rows, err := e.db.QueryContext(ctx, `WITH sums AS ( SELECT (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM FROM dual ) SELECT t.TABLESPACE_NAME, s.TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t, sums s `) if err != nil { diff --git a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go index 7e184eafef7..e2a046c140e 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go @@ -17,14 +17,6 @@ type usedAndFreeSpace struct { TotalUsedBytes sql.NullInt64 } -func (d *usedAndFreeSpace) hash() string { - return d.TablespaceName -} - -func (d *usedAndFreeSpace) eventKey() string { - return d.TablespaceName -} - func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) { rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - a.free_space) used, a.free_space free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) if err != nil { From 2ce18ac6903f2148551f5cc1c4a4352f87d9af20 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Tue, 4 Jun 2024 15:43:43 +0530 Subject: [PATCH 04/11] fix CI --- x-pack/metricbeat/module/oracle/tablespace/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index ae71b3cc766..9b357295853 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -41,7 +41,7 @@ func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { out = make(map[string]mapstr.M, 0) for _, dataFile := range in.dataFiles { - dataFileCopy := dataFile + dataFileCopy := dataFile m.addDataFileData(&dataFileCopy, out) } From 0873951e33d48d68155250d85251a14c10b8b87d Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Tue, 4 Jun 2024 22:03:36 +0530 Subject: [PATCH 05/11] fix CI issues --- .../metricbeat/module/oracle/tablespace/data_test.go | 12 ++++++------ .../module/oracle/tablespace/mocks_test.go | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/metricbeat/module/oracle/tablespace/data_test.go b/x-pack/metricbeat/module/oracle/tablespace/data_test.go index 02e4bdd4528..9c6990a2598 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data_test.go @@ -12,13 +12,13 @@ import ( "github.com/stretchr/testify/assert" ) -var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"used":{"bytes":9991}}}`, +var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"total":{"bytes":99999},"used":{"bytes":9991}}}`, `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/temp012017-03-02_07-54-38-075-AM.dbf","online_status":"ONLINE","size":{"bytes":999991,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"TEMP","space":{"free":{"bytes":99999},"total":{"bytes":99999},"used":{"bytes":99999}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`, - `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`} + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`, + `{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`} var notExpectedEvents = []string{`{}`, `{"foo":"bar"}`} diff --git a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go index 3a13dbed352..9f5800166af 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go +++ b/x-pack/metricbeat/module/oracle/tablespace/mocks_test.go @@ -84,9 +84,9 @@ type happyFreeSpaceData struct{} func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) { return []usedAndFreeSpace{ - {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, - {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 9991, Valid: true}}, + {TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, + {TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}}, }, nil } From 296ed244c3d3f4810863b180058f73fd0f4d06de Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Wed, 5 Jun 2024 15:03:32 +0530 Subject: [PATCH 06/11] address review comments --- x-pack/metricbeat/module/oracle/tablespace/data.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index 9b357295853..af1495dbed8 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -40,9 +40,8 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { out = make(map[string]mapstr.M, 0) - for _, dataFile := range in.dataFiles { - dataFileCopy := dataFile - m.addDataFileData(&dataFileCopy, out) + for i := range in.dataFiles { + m.addDataFileData(&in.dataFiles[i], out) } m.addUsedAndFreeSpaceData(in.freeSpace, out) From f99a0a0aa24c808acbdbd3e52474be503b6006fd Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Wed, 5 Jun 2024 16:31:12 +0530 Subject: [PATCH 07/11] address review comment --- x-pack/metricbeat/module/oracle/tablespace/data.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/oracle/tablespace/data.go b/x-pack/metricbeat/module/oracle/tablespace/data.go index af1495dbed8..82f93a060b1 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/data.go +++ b/x-pack/metricbeat/module/oracle/tablespace/data.go @@ -16,8 +16,9 @@ import ( // extract is the E of a ETL processing. Gets the data files, used/free space and temp free space data that is fetch // by doing queries to Oracle -func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (out *extractedData, err error) { - out = &extractedData{} +func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (*extractedData, error) { + out := &extractedData{} + var err error if out.dataFiles, err = extractor.dataFilesData(ctx); err != nil { return nil, fmt.Errorf("error getting data_files: %w", err) @@ -37,8 +38,8 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth // transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data // to create a Kibana/Elasticsearch friendly JSON. Data from Oracle is pretty fragmented by design so a lot of data // was necessary. Data is organized by Tablespace entity (Tablespaces might contain one or more data files) -func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) { - out = make(map[string]mapstr.M, 0) +func (m *MetricSet) transform(in *extractedData) map[string]mapstr.M { + out := make(map[string]mapstr.M, 0) for i := range in.dataFiles { m.addDataFileData(&in.dataFiles[i], out) From c90771fea29cda53670f10b9c685acdcdb220e33 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Wed, 5 Jun 2024 18:36:40 +0530 Subject: [PATCH 08/11] update changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1e189bac2f5..804cd6130ca 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -176,7 +176,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627] - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710] - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664] -- Fixed query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] +- Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] *Osquerybeat* From f0d0229dc645999c00fbc0e99c30cdc684f3d080 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Fri, 7 Jun 2024 10:58:49 +0530 Subject: [PATCH 09/11] address review comment --- x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go | 2 +- .../metricbeat/module/oracle/tablespace/used_and_free_space.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go index edf5d59be88..f4b592d2b75 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go @@ -18,7 +18,7 @@ type tempFreeSpace struct { } func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, `WITH sums AS ( SELECT (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM FROM dual ) SELECT t.TABLESPACE_NAME, s.TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t, sums s `) + rows, err := e.db.QueryContext(ctx, `SELECT t.TABLESPACE_NAME, (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t `) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } diff --git a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go index e2a046c140e..e327badcf02 100644 --- a/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go +++ b/x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go @@ -18,7 +18,7 @@ type usedAndFreeSpace struct { } func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) { - rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - a.free_space) used, a.free_space free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) + rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - NVL(a.free_space, 0)) AS used, NVL(a.free_space, 0) AS free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`) if err != nil { return nil, fmt.Errorf("error executing query: %w", err) } From 4b5160d7d0dc660332b6fde260f28e7ee7b09c80 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Wed, 12 Jun 2024 17:59:03 +0530 Subject: [PATCH 10/11] update visualization --- .../05acae50-9bf0-11e9-a61b-f742ed613c57.json | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json index cc48609a438..81dbcfa7c57 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json @@ -4,7 +4,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": {} }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "uiStateJSON": {}, "version": 1, "visState": { @@ -24,18 +24,23 @@ "fill": 0.5, "formatter": "bytes", "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Tablespace total size", + "label": "Tablespace max total size", "line_width": 1, "metrics": [ { + "agg_with": "avg", "field": "oracle.tablespace.space.used.bytes", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" }, { + "agg_with": "avg", + "colors": [ + "#68BC00" + ], "field": "oracle.tablespace.space.free.bytes", "id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4", - "type": "avg" + "type": "max" }, { "id": "2cf57800-8b54-41fa-a877-159b49699a50", From d092845d610a8435d62dd34042e364d777d89a72 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Thu, 13 Jun 2024 16:15:16 +0530 Subject: [PATCH 11/11] update visualization name --- .../7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json | 2 +- .../7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json index f093d13a3e4..216971d16b5 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/862e2c20-9bf0-11e9-a61b-f742ed613c57.json @@ -46,7 +46,7 @@ }, "panelIndex": "2", "panelRefName": "panel_2", - "title": "Tablespace Total Size", + "title": "Tablespace Max Total Size", "type": "visualization", "version": "8.0.0-SNAPSHOT" }, diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json index 81dbcfa7c57..27de5af579a 100644 --- a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json @@ -75,7 +75,7 @@ "type": "timeseries", "use_kibana_indexes": false }, - "title": "Tablespace Total Size [Metricbeat Oracle]", + "title": "Tablespace Max Total Size [Metricbeat Oracle]", "type": "metrics" } },