From 47bb8d05d16e86fedd043f325e25283515f01dba Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Thu, 27 Oct 2022 09:01:37 -0700 Subject: [PATCH 1/6] [Reporting/CSV Export] _id field can not be formatted (#143807) (cherry picked from commit 069937d92f2e2c98e12d62e231f67158a2664e5a) # Conflicts: # x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap --- .../generate_csv/generate_csv.ts | 16 ++- .../reporting/big_int_id_field/data.json.gz | Bin 0 -> 172 bytes .../reporting/big_int_id_field/mappings.json | 25 +++++ .../reporting/big_int_id_field.json | 96 ++++++++++++++++++ .../download_csv_dashboard.ts | 79 ++++++++++++++ 5 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 x-pack/test/functional/es_archives/reporting/big_int_id_field/data.json.gz create mode 100644 x-pack/test/functional/es_archives/reporting/big_int_id_field/mappings.json create mode 100644 x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 7039de865cd3e..d5db070bdf16c 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -148,11 +148,16 @@ export class CsvGenerator { cell = '-'; } - try { - // expected values are a string of JSON where the value(s) is in an array - cell = JSON.parse(cell); - } catch (e) { - // ignore + const isIdField = tableColumn === '_id'; // _id field can not be formatted or mutated + if (!isIdField) { + try { + // unwrap the value + // expected values are a string of JSON where the value(s) is in an array + // examples: "[""Jan 1, 2020 @ 04:00:00.000""]","[""username""]" + cell = JSON.parse(cell); + } catch (e) { + // ignore + } } // We have to strip singular array values out of their array wrapper, @@ -357,6 +362,7 @@ export class CsvGenerator { break; // empty report with just the header } + // FIXME: make tabifyDocs handle the formatting, to get the same formatting logic as Discover? const formatters = this.getFormatters(table); await this.generateRows(columns, table, builder, formatters, settings); diff --git a/x-pack/test/functional/es_archives/reporting/big_int_id_field/data.json.gz b/x-pack/test/functional/es_archives/reporting/big_int_id_field/data.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..c42d21903c912668b99e5fbd14340e9f6b5c486f GIT binary patch literal 172 zcmV;d08{@TiwFpOPFQ0C17u-zVJ>QOZ*Bmq=2B2lDyb|;RkBi0O36=F(g6vSCFYcZ zM5=)tAU87wBx__~WMp7zU~Xst24)5(V8tN$yp+@mkdl(r;*$8(oW$ai%w(8kaeir0 zGQu#Bb(y(9b&0tJP*V*I3=IqojX{QLgLqJ#K$Sps#fj;u@h}Ja2PlE1YPmqTmW!*J a0_U3!ZReW}ZReX0xbp$ahkr`@0ssIrL`dlX literal 0 HcmV?d00001 diff --git a/x-pack/test/functional/es_archives/reporting/big_int_id_field/mappings.json b/x-pack/test/functional/es_archives/reporting/big_int_id_field/mappings.json new file mode 100644 index 0000000000000..d2ee24696e0f1 --- /dev/null +++ b/x-pack/test/functional/es_archives/reporting/big_int_id_field/mappings.json @@ -0,0 +1,25 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "test_elastic", + "mappings": { + "properties": { + "timestamp": { + "format": "yyyyMMddHHmmss||yyyyMMddHHmmssZ||strict_date_optional_time||epoch_millis", + "type": "date" + }, + "message_type": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json b/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json new file mode 100644 index 0000000000000..770758f52d0d3 --- /dev/null +++ b/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json @@ -0,0 +1,96 @@ +{ + "attributes": { + "fieldAttrs": "{}", + "fieldFormatMap": "{}", + "fields": "[]", + "name": "test_elastic*", + "runtimeFieldMap": "{}", + "sourceFilters": "[]", + "timeFieldName": "timestamp", + "title": "test_elastic*", + "typeMeta": "{}" + }, + "coreMigrationVersion": "8.6.0", + "created_at": "2022-10-25T20:55:46.970Z", + "id": "c424ce04-f440-4f48-aa0c-534da84d06f6", + "migrationVersion": { + "index-pattern": "8.0.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2022-10-25T20:55:46.970Z", + "version": "WzIxOCwxXQ==" +} + +{ + "attributes": { + "columns": [], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "timestamp", + "desc" + ] + ], + "timeRestore": false, + "title": "testsearch" + }, + "coreMigrationVersion": "8.6.0", + "created_at": "2022-10-25T20:57:39.872Z", + "id": "a984aeb0-54a7-11ed-b3f3-41d5096a3cfd", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "c424ce04-f440-4f48-aa0c-534da84d06f6", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-10-25T20:57:39.872Z", + "version": "WzI2MCwxXQ==" +} + +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "optionsJSON": "{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}", + "panelsJSON": "[{\"version\":\"8.6.0\",\"type\":\"search\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":18,\"i\":\"7307be50-603d-4091-b4b9-e76a96c6a33a\"},\"panelIndex\":\"7307be50-603d-4091-b4b9-e76a96c6a33a\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7307be50-603d-4091-b4b9-e76a96c6a33a\"}]", + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-15y", + "timeRestore": true, + "timeTo": "2022-10-30T00:00:00.000Z", + "title": "rbbaf", + "version": 1 + }, + "coreMigrationVersion": "8.6.0", + "created_at": "2022-10-25T21:01:17.780Z", + "id": "b78b1350-54a7-11ed-b3f3-41d5096a3cfd", + "migrationVersion": { + "dashboard": "8.6.0" + }, + "references": [ + { + "id": "a984aeb0-54a7-11ed-b3f3-41d5096a3cfd", + "name": "7307be50-603d-4091-b4b9-e76a96c6a33a:panel_7307be50-603d-4091-b4b9-e76a96c6a33a", + "type": "search" + } + ], + "type": "dashboard", + "updated_at": "2022-10-25T21:01:17.780Z", + "version": "WzMzNiwxXQ==" +} \ No newline at end of file diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts index c0b2bafa6fcb2..78a99d766830e 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts @@ -419,6 +419,85 @@ export default function ({ getService }: FtrProviderContext) { }); }); + describe('_id field is a big integer', () => { + before(async () => { + await Promise.all([ + esArchiver.load('x-pack/test/functional/es_archives/reporting/big_int_id_field'), + kibanaServer.importExport.load( + 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' + ), + ]); + }); + + after(async () => { + await Promise.all([ + esArchiver.unload('x-pack/test/functional/es_archives/reporting/big_int_id_field'), + kibanaServer.importExport.unload( + 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' + ), + ]); + }); + it('passes through the value without mutation', async () => { + const { text } = (await generateAPI.getCSVFromSearchSource( + getMockJobParams({ + browserTimezone: 'UTC', + version: '8.6.0', + searchSource: { + query: { query: '', language: 'kuery' }, + fields: [{ field: '*', include_unmapped: 'true' }], + index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + sort: [{ timestamp: 'desc' }], + filter: [ + { + meta: { + index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + params: {}, + field: 'timestamp', + }, + query: { + range: { + timestamp: { + format: 'strict_date_optional_time', + gte: '2007-10-25T21:18:23.905Z', + lte: '2022-10-30T00:00:00.000Z', + }, + }, + }, + }, + ], + parent: { + query: { query: '', language: 'kuery' }, + filter: [], + parent: { + filter: [ + { + meta: { + index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + params: {}, + field: 'timestamp', + }, + query: { + range: { + timestamp: { + format: 'strict_date_optional_time', + gte: '2007-10-25T21:18:23.905Z', + lte: '2022-10-30T00:00:00.000Z', + }, + }, + }, + }, + ], + }, + }, + }, + columns: [], + title: 'testsearch', + }) + )) as supertest.Response; + expectSnapshot(text).toMatch(); + }); + }); + describe('validation', () => { it('Return a 404', async () => { const { body } = (await generateAPI.getCSVFromSearchSource( From 6d78f54c11134e68c823823f27c23048b9e84710 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 8 Feb 2023 11:11:59 -0700 Subject: [PATCH 2/6] fix backport --- .../reporting/big_int_id_field.json | 89 ++----------------- .../__snapshots__/download_csv_dashboard.snap | 18 ++++ .../download_csv_dashboard.ts | 59 +++++++----- 3 files changed, 60 insertions(+), 106 deletions(-) diff --git a/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json b/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json index 770758f52d0d3..063c9a7d5d2d5 100644 --- a/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json +++ b/x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field.json @@ -1,96 +1,19 @@ { "attributes": { "fieldAttrs": "{}", - "fieldFormatMap": "{}", "fields": "[]", - "name": "test_elastic*", "runtimeFieldMap": "{}", - "sourceFilters": "[]", "timeFieldName": "timestamp", - "title": "test_elastic*", + "title": "test_elastic", "typeMeta": "{}" }, - "coreMigrationVersion": "8.6.0", - "created_at": "2022-10-25T20:55:46.970Z", - "id": "c424ce04-f440-4f48-aa0c-534da84d06f6", + "coreMigrationVersion": "7.17.10", + "id": "0b3ad420-a7d9-11ed-b7bc-4b80fc2e7c64", "migrationVersion": { - "index-pattern": "8.0.0" + "index-pattern": "7.11.0" }, "references": [], "type": "index-pattern", - "updated_at": "2022-10-25T20:55:46.970Z", - "version": "WzIxOCwxXQ==" + "updated_at": "2023-02-08T17:50:15.652Z", + "version": "WzExNiwyXQ==" } - -{ - "attributes": { - "columns": [], - "description": "", - "grid": {}, - "hideChart": false, - "isTextBasedQuery": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - }, - "sort": [ - [ - "timestamp", - "desc" - ] - ], - "timeRestore": false, - "title": "testsearch" - }, - "coreMigrationVersion": "8.6.0", - "created_at": "2022-10-25T20:57:39.872Z", - "id": "a984aeb0-54a7-11ed-b3f3-41d5096a3cfd", - "migrationVersion": { - "search": "8.0.0" - }, - "references": [ - { - "id": "c424ce04-f440-4f48-aa0c-534da84d06f6", - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern" - } - ], - "type": "search", - "updated_at": "2022-10-25T20:57:39.872Z", - "version": "WzI2MCwxXQ==" -} - -{ - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "optionsJSON": "{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}", - "panelsJSON": "[{\"version\":\"8.6.0\",\"type\":\"search\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":18,\"i\":\"7307be50-603d-4091-b4b9-e76a96c6a33a\"},\"panelIndex\":\"7307be50-603d-4091-b4b9-e76a96c6a33a\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7307be50-603d-4091-b4b9-e76a96c6a33a\"}]", - "refreshInterval": { - "pause": true, - "value": 0 - }, - "timeFrom": "now-15y", - "timeRestore": true, - "timeTo": "2022-10-30T00:00:00.000Z", - "title": "rbbaf", - "version": 1 - }, - "coreMigrationVersion": "8.6.0", - "created_at": "2022-10-25T21:01:17.780Z", - "id": "b78b1350-54a7-11ed-b3f3-41d5096a3cfd", - "migrationVersion": { - "dashboard": "8.6.0" - }, - "references": [ - { - "id": "a984aeb0-54a7-11ed-b3f3-41d5096a3cfd", - "name": "7307be50-603d-4091-b4b9-e76a96c6a33a:panel_7307be50-603d-4091-b4b9-e76a96c6a33a", - "type": "search" - } - ], - "type": "dashboard", - "updated_at": "2022-10-25T21:01:17.780Z", - "version": "WzMzNiwxXQ==" -} \ No newline at end of file diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap index f642e2d5c2784..6661b845a0b43 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap +++ b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap @@ -1,5 +1,23 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Reporting APIs CSV Generation from SearchSource _id field is a big integer, passes through the value without mutation (ES 7) 1`] = ` +"_id,_index,_score,_type,message_type,timestamp +202209071000000604,test_elastic,-,_doc,OP,Jan 1, 2020 @ 11:00:00.000 +202209071000000605,test_elastic,-,_doc,OP,Jan 1, 2020 @ 11:00:00.000 +202209071000000606,test_elastic,-,_doc,OP,Jan 1, 2020 @ 11:00:00.000 +202209071000000607,test_elastic,-,_doc,OP,Jan 1, 2020 @ 11:00:00.000 +" +`; + +exports[`Reporting APIs CSV Generation from SearchSource _id field is a big integer, passes through the value without mutation (ES 8) 1`] = ` +"_id,_index,_score,_type,message_type,timestamp +202209071000000604,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 +202209071000000605,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 +202209071000000606,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 +202209071000000607,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 +" +`; + exports[`Reporting APIs CSV Generation from SearchSource Exports CSV with all fields when using defaults (ES 7) 1`] = ` "_id,_index,_score,_type,category,category.keyword,currency,customer_first_name,customer_first_name.keyword,customer_full_name,customer_full_name.keyword,customer_gender,customer_id,customer_last_name,customer_last_name.keyword,customer_phone,day_of_week,day_of_week_i,email,geoip.city_name,geoip.continent_name,geoip.country_iso_code,geoip.location,geoip.region_name,manufacturer,manufacturer.keyword,order_date,order_id,products._id,products._id.keyword,products.base_price,products.base_unit_price,products.category,products.category.keyword,products.created_on,products.discount_amount,products.discount_percentage,products.manufacturer,products.manufacturer.keyword,products.min_price,products.price,products.product_id,products.product_name,products.product_name.keyword,products.quantity,products.sku,products.tax_amount,products.taxful_price,products.taxless_price,products.unit_discount_amount,sku,taxful_total_price,taxless_total_price,total_quantity,total_unique_products,type,user 9AMtOW0BH63Xcmy432DJ,ecommerce,-,_doc,Men's Clothing,Men's Clothing,EUR,Boris,Boris,Boris Bradley,Boris Bradley,MALE,36,Bradley,Bradley,(empty),Wednesday,2,boris@bradley-family.zzz,-,Europe,GB,{ diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts index 78a99d766830e..d5629add3cfb4 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts @@ -419,38 +419,30 @@ export default function ({ getService }: FtrProviderContext) { }); }); - describe('_id field is a big integer', () => { + describe('_id field is a big integer, passes through the value without mutation', () => { + let resText: string; before(async () => { - await Promise.all([ - esArchiver.load('x-pack/test/functional/es_archives/reporting/big_int_id_field'), - kibanaServer.importExport.load( - 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' - ), - ]); - }); - - after(async () => { - await Promise.all([ - esArchiver.unload('x-pack/test/functional/es_archives/reporting/big_int_id_field'), - kibanaServer.importExport.unload( - 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' - ), - ]); - }); - it('passes through the value without mutation', async () => { - const { text } = (await generateAPI.getCSVFromSearchSource( + await esArchiver.load('x-pack/test/functional/es_archives/reporting/big_int_id_field'); + await kibanaServer.importExport.load( + 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' + ); + const { + status: resStatus, + type: resType, + text, + } = (await generateAPI.getCSVFromSearchSource( getMockJobParams({ browserTimezone: 'UTC', version: '8.6.0', searchSource: { query: { query: '', language: 'kuery' }, fields: [{ field: '*', include_unmapped: 'true' }], - index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + index: '0b3ad420-a7d9-11ed-b7bc-4b80fc2e7c64', sort: [{ timestamp: 'desc' }], filter: [ { meta: { - index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + index: '0b3ad420-a7d9-11ed-b7bc-4b80fc2e7c64', params: {}, field: 'timestamp', }, @@ -472,7 +464,7 @@ export default function ({ getService }: FtrProviderContext) { filter: [ { meta: { - index: 'c424ce04-f440-4f48-aa0c-534da84d06f6', + index: '0b3ad420-a7d9-11ed-b7bc-4b80fc2e7c64', params: {}, field: 'timestamp', }, @@ -494,7 +486,28 @@ export default function ({ getService }: FtrProviderContext) { title: 'testsearch', }) )) as supertest.Response; - expectSnapshot(text).toMatch(); + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + + resText = text; + }); + + after(async () => { + await Promise.all([ + esArchiver.unload('x-pack/test/functional/es_archives/reporting/big_int_id_field'), + kibanaServer.importExport.unload( + 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' + ), + ]); + }); + + itIfEs7('(ES 7)', async () => { + expectSnapshot(resText).toMatch(); + }); + + itIfEs8('(ES 8)', async () => { + expectSnapshot(resText).toMatch(); }); }); From 8ae844dfc66049852824702ba59bef123b429909 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 8 Feb 2023 12:44:47 -0700 Subject: [PATCH 3/6] fix test --- .../reporting_and_security/download_csv_dashboard.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts index d5629add3cfb4..61431457d276b 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts @@ -426,6 +426,11 @@ export default function ({ getService }: FtrProviderContext) { await kibanaServer.importExport.load( 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' ); + await kibanaServer.uiSettings.update({ + 'csv:quoteValues': false, + 'dateFormat:tz': 'UTC', + defaultIndex: 'logstash-*', + }); const { status: resStatus, type: resType, From c77ee5378632544289a27eca16008970341a6fcc Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 8 Feb 2023 14:07:05 -0700 Subject: [PATCH 4/6] fix tests --- .../__snapshots__/download_csv_dashboard.snap | 9 - .../download_csv_dashboard.ts | 162 +++++++++--------- 2 files changed, 81 insertions(+), 90 deletions(-) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap index 6661b845a0b43..c8501a2f3677e 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap +++ b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap @@ -9,15 +9,6 @@ exports[`Reporting APIs CSV Generation from SearchSource _id field is a big inte " `; -exports[`Reporting APIs CSV Generation from SearchSource _id field is a big integer, passes through the value without mutation (ES 8) 1`] = ` -"_id,_index,_score,_type,message_type,timestamp -202209071000000604,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 -202209071000000605,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 -202209071000000606,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 -202209071000000607,test_elastic,-,-,OP,Jan 1, 2020 @ 11:00:00.000 -" -`; - exports[`Reporting APIs CSV Generation from SearchSource Exports CSV with all fields when using defaults (ES 7) 1`] = ` "_id,_index,_score,_type,category,category.keyword,currency,customer_first_name,customer_first_name.keyword,customer_full_name,customer_full_name.keyword,customer_gender,customer_id,customer_last_name,customer_last_name.keyword,customer_phone,day_of_week,day_of_week_i,email,geoip.city_name,geoip.continent_name,geoip.country_iso_code,geoip.location,geoip.region_name,manufacturer,manufacturer.keyword,order_date,order_id,products._id,products._id.keyword,products.base_price,products.base_unit_price,products.category,products.category.keyword,products.created_on,products.discount_amount,products.discount_percentage,products.manufacturer,products.manufacturer.keyword,products.min_price,products.price,products.product_id,products.product_name,products.product_name.keyword,products.quantity,products.sku,products.tax_amount,products.taxful_price,products.taxless_price,products.unit_discount_amount,sku,taxful_total_price,taxless_total_price,total_quantity,total_unique_products,type,user 9AMtOW0BH63Xcmy432DJ,ecommerce,-,_doc,Men's Clothing,Men's Clothing,EUR,Boris,Boris,Boris Bradley,Boris Bradley,MALE,36,Bradley,Bradley,(empty),Wednesday,2,boris@bradley-family.zzz,-,Europe,GB,{ diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts index 61431457d276b..db9b2081b7464 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/download_csv_dashboard.ts @@ -419,18 +419,95 @@ export default function ({ getService }: FtrProviderContext) { }); }); + describe('validation', () => { + it('Return a 404', async () => { + const { body } = (await generateAPI.getCSVFromSearchSource( + getMockJobParams({ + searchSource: { + index: 'gobbledygook', + }, + }) + )) as supertest.Response; + const expectedBody = { + error: 'Not Found', + message: 'Saved object [index-pattern/gobbledygook] not found', + statusCode: 404, + }; + expect(body).to.eql(expectedBody); + }); + + // NOTE: this test requires having the test server run with `xpack.reporting.csv.maxSizeBytes=6000` + describe(`Searches large amount of data, stops at Max Size Reached`, () => { + let resText: string; + before(async () => { + await reportingAPI.initEcommerce(); + + const { + status: resStatus, + type: resType, + text, + } = (await generateAPI.getCSVFromSearchSource( + getMockJobParams({ + searchSource: { + version: true, + query: { query: '', language: 'kuery' }, + index: '5193f870-d861-11e9-a311-0fa548c5f953', + sort: [{ order_date: 'desc' }], + fields: ['*'], + filter: [], + parent: { + query: { language: 'kuery', query: '' }, + filter: [], + parent: { + filter: [ + { + meta: { index: '5193f870-d861-11e9-a311-0fa548c5f953', params: {} }, + range: { + order_date: { + gte: '2019-03-23T00:00:00.000Z', + lte: '2019-10-04T00:00:00.000Z', + format: 'strict_date_optional_time', + }, + }, + }, + ], + }, + }, + }, + browserTimezone: 'UTC', + title: 'Ecommerce Data', + }) + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + + await reportingAPI.teardownEcommerce(); + resText = text; + }); + + itIfEs7('(ES 7)', async () => { + expectSnapshot(resText).toMatch(); + }); + + itIfEs8('(ES 8)', async () => { + expectSnapshot(resText).toMatch(); + }); + }); + }); + describe('_id field is a big integer, passes through the value without mutation', () => { let resText: string; before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/reporting/big_int_id_field'); - await kibanaServer.importExport.load( - 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' - ); await kibanaServer.uiSettings.update({ 'csv:quoteValues': false, 'dateFormat:tz': 'UTC', defaultIndex: 'logstash-*', }); + await esArchiver.load('x-pack/test/functional/es_archives/reporting/big_int_id_field'); + await kibanaServer.importExport.load( + 'x-pack/test/functional/fixtures/kbn_archiver/reporting/big_int_id_field' + ); const { status: resStatus, type: resType, @@ -515,82 +592,5 @@ export default function ({ getService }: FtrProviderContext) { expectSnapshot(resText).toMatch(); }); }); - - describe('validation', () => { - it('Return a 404', async () => { - const { body } = (await generateAPI.getCSVFromSearchSource( - getMockJobParams({ - searchSource: { - index: 'gobbledygook', - }, - }) - )) as supertest.Response; - const expectedBody = { - error: 'Not Found', - message: 'Saved object [index-pattern/gobbledygook] not found', - statusCode: 404, - }; - expect(body).to.eql(expectedBody); - }); - - // NOTE: this test requires having the test server run with `xpack.reporting.csv.maxSizeBytes=6000` - describe(`Searches large amount of data, stops at Max Size Reached`, () => { - let resText: string; - before(async () => { - await reportingAPI.initEcommerce(); - - const { - status: resStatus, - type: resType, - text, - } = (await generateAPI.getCSVFromSearchSource( - getMockJobParams({ - searchSource: { - version: true, - query: { query: '', language: 'kuery' }, - index: '5193f870-d861-11e9-a311-0fa548c5f953', - sort: [{ order_date: 'desc' }], - fields: ['*'], - filter: [], - parent: { - query: { language: 'kuery', query: '' }, - filter: [], - parent: { - filter: [ - { - meta: { index: '5193f870-d861-11e9-a311-0fa548c5f953', params: {} }, - range: { - order_date: { - gte: '2019-03-23T00:00:00.000Z', - lte: '2019-10-04T00:00:00.000Z', - format: 'strict_date_optional_time', - }, - }, - }, - ], - }, - }, - }, - browserTimezone: 'UTC', - title: 'Ecommerce Data', - }) - )) as supertest.Response; - - expect(resStatus).to.eql(200); - expect(resType).to.eql('text/csv'); - - await reportingAPI.teardownEcommerce(); - resText = text; - }); - - itIfEs7('(ES 7)', async () => { - expectSnapshot(resText).toMatch(); - }); - - itIfEs8('(ES 8)', async () => { - expectSnapshot(resText).toMatch(); - }); - }); - }); }); } From 360f4dec77fc839323629ed1a3487f1fe4a7afd5 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 8 Feb 2023 17:51:42 -0700 Subject: [PATCH 5/6] simplify for 7.17 --- .../export_types/csv_searchsource/generate_csv/generate_csv.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index d5db070bdf16c..21d7ceb07c8dc 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -362,7 +362,6 @@ export class CsvGenerator { break; // empty report with just the header } - // FIXME: make tabifyDocs handle the formatting, to get the same formatting logic as Discover? const formatters = this.getFormatters(table); await this.generateRows(columns, table, builder, formatters, settings); From 327e18889bf674b921a6ee487bdaf92248bb63ab Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Thu, 9 Feb 2023 15:32:21 -0700 Subject: [PATCH 6/6] fix csv searchsource tests --- .../reporting_and_security/generate_csv_discover.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/generate_csv_discover.ts b/x-pack/test/reporting_api_integration/reporting_and_security/generate_csv_discover.ts index 1f99bac389a9c..7a4eb58f28fab 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/generate_csv_discover.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/generate_csv_discover.ts @@ -12,6 +12,8 @@ import { FtrProviderContext } from '../ftr_provider_context'; // eslint-disable-next-line import/no-default-export export default function ({ getService }: FtrProviderContext) { + const log = getService('log'); + const kibanaServer = getService('kibanaServer'); const reportingAPI = getService('reportingAPI'); const esVersion = getService('esVersion'); @@ -21,6 +23,11 @@ export default function ({ getService }: FtrProviderContext) { before(async () => { await reportingAPI.initEcommerce(); + log.info(`updating Advanced Settings`); + await kibanaServer.uiSettings.update({ + 'csv:quoteValues': true, + }); + const fromTime = '2019-06-20T00:00:00.000Z'; const toTime = '2019-06-24T00:00:00.000Z';