From d408694f819682d80738b7b1f2f7113424ecca7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 3 Apr 2024 17:03:10 +0200 Subject: [PATCH] refactor: add integration tests that check the HTML content of different types of pages + fix issues (#10043) --- cgi/display.pl | 4 +- cgi/export_products.pl | 2 +- cgi/import_file_process.pl | 8 +- cgi/import_file_select_format.pl | 6 +- cgi/import_file_upload.pl | 5 +- cgi/import_photos_upload.pl | 2 +- ...roducts_categories_from_public_database.pl | 2 +- cgi/org.pl | 8 +- cgi/product_image.pl | 6 +- cgi/product_image_upload.pl | 4 +- cgi/product_jqm_multilingual.pl | 2 +- cgi/product_multilingual.pl | 21 +- cgi/remove_products.pl | 6 +- cgi/reset_password.pl | 6 +- cgi/search.pl | 4 +- cgi/user.pl | 8 +- cpanfile | 1 + lib/ProductOpener/APITest.pm | 20 +- lib/ProductOpener/Display.pm | 192 +- lib/ProductOpener/Tags.pm | 1 + lib/ProductOpener/Test.pm | 44 +- lib/ProductOpener/Users.pm | 8 +- taxonomies/languages.txt | 70 +- .../product/includes/edit_history.tt.html | 2 +- .../product_edit_form_packagings.tt.html | 6 - templates/web/pages/tag/tag.tt.html | 8 +- .../pages/user_form/user_form_page.tt.html | 32 +- tests/integration/api_v3_product_write.t | 2 +- tests/integration/cors.t | 19 +- .../get-auth-bad-user-password.html | 507 ++ .../post-product-auth-bad-user-password.html | 507 ++ .../api_v3_product_write/patch-code-none.json | 25 - ...apes-categories-mango-juice-beverages.json | 32 - .../cors/get-api-v2.json | 5 + .../export/0850032917148.json | 113 - .../export/25000044984.json | 113 - .../export/26281742.json | 113 - .../export/27096765.json | 113 - .../export/29161690.json | 113 - .../export/3173990027337.json | 113 - .../export/3250392332105.json | 113 - .../export/3256220513173.json | 113 - .../export/3259330020135.json | 113 - .../export/3270160503070.json | 113 - .../export/3451790834080.json | 113 - .../export/3564703999971.json | 113 - .../export/3661344653573.json | 113 - .../export/3760178254021.json | 113 - .../export/3770013801303.json | 113 - .../export/5050083706622.json | 113 - .../export/5410803950689.json | 113 - .../export/5601009974337.json | 113 - .../export/71464240608.json | 113 - .../export/77000001.json | 113 - .../export/7804659650035.json | 113 - .../export/80650904.json | 113 - .../export/8712423020221.json | 113 - .../export/8722700472575.json | 113 - .../export/9002355004345.json | 113 - .../export_more_fields/0850032917148.json | 139 - .../export_more_fields/25000044984.json | 139 - .../export_more_fields/26281742.json | 139 - .../export_more_fields/27096765.json | 139 - .../export_more_fields/29161690.json | 139 - .../export_more_fields/3173990027337.json | 139 - .../export_more_fields/3250392332105.json | 139 - .../export_more_fields/3256220513173.json | 139 - .../export_more_fields/3259330020135.json | 139 - .../export_more_fields/3270160503070.json | 139 - .../export_more_fields/3451790834080.json | 139 - .../export_more_fields/3564703999971.json | 139 - .../export_more_fields/3661344653573.json | 139 - .../export_more_fields/3760178254021.json | 139 - .../export_more_fields/3770013801303.json | 139 - .../export_more_fields/5050083706622.json | 139 - .../export_more_fields/5410803950689.json | 139 - .../export_more_fields/5601009974337.json | 139 - .../export_more_fields/71464240608.json | 139 - .../export_more_fields/77000001.json | 139 - .../export_more_fields/7804659650035.json | 139 - .../export_more_fields/80650904.json | 139 - .../export_more_fields/8712423020221.json | 139 - .../export_more_fields/8722700472575.json | 139 - .../export_more_fields/9002355004345.json | 139 - .../crawler-access-category-facet-page.html | 681 +++ .../crawler-access-editor-facet-page.html | 1 + .../crawler-access-list-of-tags.html | 1 + .../crawler-access-nested-facet-page.html | 1 + .../crawler-access-product-page.html | 4404 +++++++++++++++ ...r-does-not-get-facet-knowledge-panels.html | 744 +++ .../crawler-get-non-official-cc-lc.html | 1 + .../denied-crawler-access-product-page.html | 1 + ...xt-ch-it.txt => get-robots-txt-ch-it.text} | 0 ...xt => get-robots-txt-fr-pro-platform.text} | 0 ...bots-txt-fr.txt => get-robots-txt-fr.text} | 0 ...t-robots-txt-invalid-cc-lc-subdomain.text} | 0 ... => get-robots-txt-ssl-api-subdomain.text} | 0 ... => get-robots-txt-word-lc-subdomain.text} | 0 ...=> get-robots-txt-world-pro-platform.text} | 0 ...xt-world.txt => get-robots-txt-world.text} | 0 ...ormal-user-access-category-facet-page.html | 722 +++ .../normal-user-access-editor-facet-page.html | 509 ++ .../normal-user-access-list-of-tags.html | 566 ++ .../normal-user-access-nested-facet-page.html | 717 +++ .../normal-user-access-product-page.html | 4404 +++++++++++++++ ...ormal-user-get-facet-knowledge-panels.html | 785 +++ .../normal-user-get-non-official-cc-lc.html | 577 ++ .../product_read/get-existing-product.html | 4943 +++++++++++++++++ .../product_read/get-unexisting-product.html | 518 ++ ...-protected-product-web-form-moderator.html | 940 ++++ .../edit-protected-product-web-form.html | 973 ++++ .../edit-unprotected-product-web-form.html | 940 ++++ .../country-cambodia-exists-but-empty.html | 657 +++ ...ountry-doesnotexist-ingredients-apple.html | 521 ++ .../country-doesnotexist-ingredients.html | 509 ++ .../unknown_tags/country-doesnotexist.html | 521 ++ .../unknown_tags/country-france-exists.html | 713 +++ .../unknown_tags/ingredient-apple-exists.html | 776 +++ ...tyingredient-does-not-exist-and-empty.html | 521 ++ ...t-does-not-exist-but-not-empty-labels.html | 597 ++ ...gredient-does-not-exist-but-not-empty.html | 713 +++ .../unknown_tags/unknown-product.html | 518 ++ .../web_html/fr-brands.html | 562 ++ .../web_html/fr-categories.html | 814 +++ .../web_html/fr-countries.html | 572 ++ .../web_html/fr-edit-product.html | 4069 ++++++++++++++ .../web_html/fr-index.html | 849 +++ .../web_html/fr-labels.html | 562 ++ .../web_html/fr-product-2.html | 4569 +++++++++++++++ .../web_html/fr-product.html | 4583 +++++++++++++++ .../web_html/fr-search-form.html | 2142 +++++++ .../web_html/fr-search-results.html | 655 +++ .../web_html/user-register.html | 1725 ++++++ .../web_html/world-brands.html | 570 ++ .../web_html/world-categories.html | 973 ++++ .../web_html/world-countries.html | 584 ++ .../web_html/world-edit-product.html | 4069 ++++++++++++++ .../web_html/world-index-signedin.html | 952 ++++ .../web_html/world-index.html | 929 ++++ .../web_html/world-label-organic.html | 828 +++ .../web_html/world-labels.html | 566 ++ .../web_html/world-product-not-found.html | 518 ++ .../web_html/world-product.html | 4615 +++++++++++++++ .../web_html/world-search-form.html | 2142 +++++++ .../web_html/world-search-results.html | 694 +++ tests/integration/page_crawler.t | 2 +- tests/integration/web_html.t | 470 ++ 147 files changed, 68719 insertions(+), 6644 deletions(-) create mode 100644 tests/integration/expected_test_results/api_v2_product_read/get-auth-bad-user-password.html create mode 100644 tests/integration/expected_test_results/api_v2_product_write/post-product-auth-bad-user-password.html delete mode 100644 tests/integration/expected_test_results/api_v3_product_write/patch-code-none.json delete mode 100644 tests/integration/expected_test_results/api_v3_taxonomy_suggestions/packaging-shapes-categories-mango-juice-beverages.json create mode 100644 tests/integration/expected_test_results/cors/get-api-v2.json delete mode 100644 tests/integration/expected_test_results/export/0850032917148.json delete mode 100644 tests/integration/expected_test_results/export/25000044984.json delete mode 100644 tests/integration/expected_test_results/export/26281742.json delete mode 100644 tests/integration/expected_test_results/export/27096765.json delete mode 100644 tests/integration/expected_test_results/export/29161690.json delete mode 100644 tests/integration/expected_test_results/export/3173990027337.json delete mode 100644 tests/integration/expected_test_results/export/3250392332105.json delete mode 100644 tests/integration/expected_test_results/export/3256220513173.json delete mode 100644 tests/integration/expected_test_results/export/3259330020135.json delete mode 100644 tests/integration/expected_test_results/export/3270160503070.json delete mode 100644 tests/integration/expected_test_results/export/3451790834080.json delete mode 100644 tests/integration/expected_test_results/export/3564703999971.json delete mode 100644 tests/integration/expected_test_results/export/3661344653573.json delete mode 100644 tests/integration/expected_test_results/export/3760178254021.json delete mode 100644 tests/integration/expected_test_results/export/3770013801303.json delete mode 100644 tests/integration/expected_test_results/export/5050083706622.json delete mode 100644 tests/integration/expected_test_results/export/5410803950689.json delete mode 100644 tests/integration/expected_test_results/export/5601009974337.json delete mode 100644 tests/integration/expected_test_results/export/71464240608.json delete mode 100644 tests/integration/expected_test_results/export/77000001.json delete mode 100644 tests/integration/expected_test_results/export/7804659650035.json delete mode 100644 tests/integration/expected_test_results/export/80650904.json delete mode 100644 tests/integration/expected_test_results/export/8712423020221.json delete mode 100644 tests/integration/expected_test_results/export/8722700472575.json delete mode 100644 tests/integration/expected_test_results/export/9002355004345.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/0850032917148.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/25000044984.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/26281742.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/27096765.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/29161690.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3173990027337.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3250392332105.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3256220513173.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3259330020135.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3270160503070.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3451790834080.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3564703999971.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3661344653573.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3760178254021.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/3770013801303.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/5050083706622.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/5410803950689.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/5601009974337.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/71464240608.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/77000001.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/7804659650035.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/80650904.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/8712423020221.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/8722700472575.json delete mode 100644 tests/integration/expected_test_results/export_more_fields/9002355004345.json create mode 100644 tests/integration/expected_test_results/page_crawler/crawler-access-category-facet-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/crawler-access-editor-facet-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/crawler-access-list-of-tags.html create mode 100644 tests/integration/expected_test_results/page_crawler/crawler-access-nested-facet-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/crawler-access-product-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/crawler-does-not-get-facet-knowledge-panels.html create mode 100644 tests/integration/expected_test_results/page_crawler/crawler-get-non-official-cc-lc.html create mode 100644 tests/integration/expected_test_results/page_crawler/denied-crawler-access-product-page.html rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-ch-it.txt => get-robots-txt-ch-it.text} (100%) rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-fr-pro-platform.txt => get-robots-txt-fr-pro-platform.text} (100%) rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-fr.txt => get-robots-txt-fr.text} (100%) rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-invalid-cc-lc-subdomain.txt => get-robots-txt-invalid-cc-lc-subdomain.text} (100%) rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-ssl-api-subdomain.txt => get-robots-txt-ssl-api-subdomain.text} (100%) rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-word-lc-subdomain.txt => get-robots-txt-word-lc-subdomain.text} (100%) rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-world-pro-platform.txt => get-robots-txt-world-pro-platform.text} (100%) rename tests/integration/expected_test_results/page_crawler/{get-robots-txt-world.txt => get-robots-txt-world.text} (100%) create mode 100644 tests/integration/expected_test_results/page_crawler/normal-user-access-category-facet-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/normal-user-access-editor-facet-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/normal-user-access-list-of-tags.html create mode 100644 tests/integration/expected_test_results/page_crawler/normal-user-access-nested-facet-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/normal-user-access-product-page.html create mode 100644 tests/integration/expected_test_results/page_crawler/normal-user-get-facet-knowledge-panels.html create mode 100644 tests/integration/expected_test_results/page_crawler/normal-user-get-non-official-cc-lc.html create mode 100644 tests/integration/expected_test_results/product_read/get-existing-product.html create mode 100644 tests/integration/expected_test_results/product_read/get-unexisting-product.html create mode 100644 tests/integration/expected_test_results/protected_product/edit-protected-product-web-form-moderator.html create mode 100644 tests/integration/expected_test_results/protected_product/edit-protected-product-web-form.html create mode 100644 tests/integration/expected_test_results/protected_product/edit-unprotected-product-web-form.html create mode 100644 tests/integration/expected_test_results/unknown_tags/country-cambodia-exists-but-empty.html create mode 100644 tests/integration/expected_test_results/unknown_tags/country-doesnotexist-ingredients-apple.html create mode 100644 tests/integration/expected_test_results/unknown_tags/country-doesnotexist-ingredients.html create mode 100644 tests/integration/expected_test_results/unknown_tags/country-doesnotexist.html create mode 100644 tests/integration/expected_test_results/unknown_tags/country-france-exists.html create mode 100644 tests/integration/expected_test_results/unknown_tags/ingredient-apple-exists.html create mode 100644 tests/integration/expected_test_results/unknown_tags/ingredient-someunknownandemptyingredient-does-not-exist-and-empty.html create mode 100644 tests/integration/expected_test_results/unknown_tags/ingredient-someunknowningredient-does-not-exist-but-not-empty-labels.html create mode 100644 tests/integration/expected_test_results/unknown_tags/ingredient-someunknowningredient-does-not-exist-but-not-empty.html create mode 100644 tests/integration/expected_test_results/unknown_tags/unknown-product.html create mode 100644 tests/integration/expected_test_results/web_html/fr-brands.html create mode 100644 tests/integration/expected_test_results/web_html/fr-categories.html create mode 100644 tests/integration/expected_test_results/web_html/fr-countries.html create mode 100644 tests/integration/expected_test_results/web_html/fr-edit-product.html create mode 100644 tests/integration/expected_test_results/web_html/fr-index.html create mode 100644 tests/integration/expected_test_results/web_html/fr-labels.html create mode 100644 tests/integration/expected_test_results/web_html/fr-product-2.html create mode 100644 tests/integration/expected_test_results/web_html/fr-product.html create mode 100644 tests/integration/expected_test_results/web_html/fr-search-form.html create mode 100644 tests/integration/expected_test_results/web_html/fr-search-results.html create mode 100644 tests/integration/expected_test_results/web_html/user-register.html create mode 100644 tests/integration/expected_test_results/web_html/world-brands.html create mode 100644 tests/integration/expected_test_results/web_html/world-categories.html create mode 100644 tests/integration/expected_test_results/web_html/world-countries.html create mode 100644 tests/integration/expected_test_results/web_html/world-edit-product.html create mode 100644 tests/integration/expected_test_results/web_html/world-index-signedin.html create mode 100644 tests/integration/expected_test_results/web_html/world-index.html create mode 100644 tests/integration/expected_test_results/web_html/world-label-organic.html create mode 100644 tests/integration/expected_test_results/web_html/world-labels.html create mode 100644 tests/integration/expected_test_results/web_html/world-product-not-found.html create mode 100644 tests/integration/expected_test_results/web_html/world-product.html create mode 100644 tests/integration/expected_test_results/web_html/world-search-form.html create mode 100644 tests/integration/expected_test_results/web_html/world-search-results.html create mode 100644 tests/integration/web_html.t diff --git a/cgi/display.pl b/cgi/display.pl index ab4c62490201d..702453fe2e7d1 100755 --- a/cgi/display.pl +++ b/cgi/display.pl @@ -105,7 +105,7 @@ if (defined $request_ref->{error_message}) { $log->debug("analyze_request error", {request_ref => $request_ref}); - display_error($request_ref->{error_message}, $request_ref->{status_code}); + display_error($request_ref, $request_ref->{error_message}, $request_ref->{status_code}); $log->debug("analyze_request error - return Apache2::Const::OK"); return Apache2::Const::OK; } @@ -139,7 +139,7 @@ ) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } if ((defined $request_ref->{api}) and (defined $request_ref->{api_action})) { diff --git a/cgi/export_products.pl b/cgi/export_products.pl index 079e4795ee931..dfadfcdf451c0 100755 --- a/cgi/export_products.pl +++ b/cgi/export_products.pl @@ -58,7 +58,7 @@ my $html = ''; if (not defined $Owner_id) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } # Require moderator status to launch the export / import process, diff --git a/cgi/import_file_process.pl b/cgi/import_file_process.pl index 6e21399db10da..244244e2cfddb 100755 --- a/cgi/import_file_process.pl +++ b/cgi/import_file_process.pl @@ -57,7 +57,7 @@ my $template_data_ref; if (not defined $Owner_id) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } my $import_files_ref = retrieve("$BASE_DIRS{IMPORT_FILES}/${Owner_id}/import_files.sto"); @@ -78,7 +78,7 @@ } else { $log->debug("File not found in import_files.sto", {file_id => $file_id}) if $log->is_debug(); - display_error_and_exit("File not found.", 404); + display_error_and_exit($request_ref, "File not found.", 404); } $log->debug("File found in import_files.sto", @@ -95,7 +95,7 @@ my $results_ref = load_csv_or_excel_file($file); if ($results_ref->{error}) { - display_error_and_exit($results_ref->{error}, 200); + display_error_and_exit($request_ref, $results_ref->{error}, 200); } my $headers_ref = $results_ref->{headers}; @@ -152,7 +152,7 @@ if ($results_ref->{error}) { $import_files_ref->{$file_id}{imports}{$import_id}{convert_error} = $results_ref->{error}; store("$BASE_DIRS{IMPORT_FILES}/${Owner_id}/import_files.sto", $import_files_ref); - display_error_and_exit($results_ref->{error}, 200); + display_error_and_exit($request_ref, $results_ref->{error}, 200); } my $args_ref = { diff --git a/cgi/import_file_select_format.pl b/cgi/import_file_select_format.pl index 95e726fb7fe7e..79ee4b492cc38 100755 --- a/cgi/import_file_select_format.pl +++ b/cgi/import_file_select_format.pl @@ -62,7 +62,7 @@ my $template_data_ref = {}; if (not defined $Owner_id) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } my $import_files_ref = retrieve("$BASE_DIRS{IMPORT_FILES}/${Owner_id}/import_files.sto"); @@ -84,7 +84,7 @@ } else { $log->debug("File not found in import_files.sto", {file_id => $file_id}) if $log->is_debug(); - display_error_and_exit("File not found.", 404); + display_error_and_exit($request_ref, "File not found.", 404); } $log->debug("File found in import_files.sto", @@ -96,7 +96,7 @@ my $results_ref = load_csv_or_excel_file($file); if ($results_ref->{error}) { - display_error_and_exit($results_ref->{error}, 200); + display_error_and_exit($request_ref, $results_ref->{error}, 200); } my $headers_ref = $results_ref->{headers}; diff --git a/cgi/import_file_upload.pl b/cgi/import_file_upload.pl index 6bf2d973e29fb..bb744981e2cf1 100755 --- a/cgi/import_file_upload.pl +++ b/cgi/import_file_upload.pl @@ -59,7 +59,7 @@ local $log->context->{action} = $action; if (not defined $Owner_id) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } if ($action eq "process") { @@ -81,7 +81,8 @@ $log->debug("processing upload form", {filename => $filename, file_id => $file_id, extension => $extension}) if $log->is_debug(); - ensure_dir_created("$BASE_DIRS{IMPORT_FILES}/${Owner_id}") or display_error_and_exit("Missing path", 503); + ensure_dir_created("$BASE_DIRS{IMPORT_FILES}/${Owner_id}") + or display_error_and_exit($request_ref, "Missing path", 503); open(my $out, ">", "$BASE_DIRS{IMPORT_FILES}/${Owner_id}/$file_id.$extension"); while (my $chunk = <$file>) { diff --git a/cgi/import_photos_upload.pl b/cgi/import_photos_upload.pl index eb50c45b50abc..b66c5538370d0 100755 --- a/cgi/import_photos_upload.pl +++ b/cgi/import_photos_upload.pl @@ -59,7 +59,7 @@ local $log->context->{action} = $action; if (not defined $Owner_id) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } else { diff --git a/cgi/import_products_categories_from_public_database.pl b/cgi/import_products_categories_from_public_database.pl index 276dac1c4a3fe..d069f4e6e7a04 100755 --- a/cgi/import_products_categories_from_public_database.pl +++ b/cgi/import_products_categories_from_public_database.pl @@ -58,7 +58,7 @@ my $template_data_ref = {}; if (not defined $Owner_id) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } if ($action eq "display") { diff --git a/cgi/org.pl b/cgi/org.pl index 1a0b586e709c2..56c5e0a8b8089 100644 --- a/cgi/org.pl +++ b/cgi/org.pl @@ -72,7 +72,7 @@ $template_data_ref->{org_does_not_exist} = 1; } else { - display_error_and_exit($Lang{error_org_does_not_exist}{$lc}, 404); + display_error_and_exit($request_ref, $Lang{error_org_does_not_exist}{$lc}, 404); } } @@ -82,7 +82,7 @@ $log->debug("user does not have permission to edit org", {orgid => $orgid, org_admins => $org_ref->{admins}, User_id => $User_id}) if $log->is_debug(); - display_error_and_exit($Lang{error_no_permission}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{error_no_permission}{$lc}, 403); } my @errors = (); @@ -95,7 +95,7 @@ $type = 'delete'; } else { - display_error_and_exit($Lang{error_no_permission}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{error_no_permission}{$lc}, 403); } } else { @@ -369,7 +369,7 @@ $template_data_ref->{result} = lang("edit_org_result"); } else { - display_error_and_exit($Lang{error_no_permission}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{error_no_permission}{$lc}, 403); } } diff --git a/cgi/product_image.pl b/cgi/product_image.pl index fbdc7c8ad772c..417ef7ef8799c 100644 --- a/cgi/product_image.pl +++ b/cgi/product_image.pl @@ -53,7 +53,7 @@ $log->debug("start", {code => $code, id => $id}) if $log->is_debug(); if (not defined $code) { - display_error_and_exit(sprintf(lang("no_product_for_barcode"), $code), 404); + display_error_and_exit($request_ref, sprintf(lang("no_product_for_barcode"), $code), 404); } my $product_id = product_id_for_owner($Owner_id, $code); @@ -61,11 +61,11 @@ my $product_ref = retrieve_product($product_id); if (not(defined $product_ref)) { - display_error_and_exit(sprintf(lang("no_product_for_barcode"), $code), 404); + display_error_and_exit($request_ref, sprintf(lang("no_product_for_barcode"), $code), 404); } if ((not(defined $product_ref->{images})) or (not(defined $product_ref->{images}{$id}))) { - display_error_and_exit(sprintf(lang("no_product_for_barcode"), $code), 404); + display_error_and_exit($request_ref, sprintf(lang("no_product_for_barcode"), $code), 404); } my $imagetext; diff --git a/cgi/product_image_upload.pl b/cgi/product_image_upload.pl index 229a4fdf1c640..57b4d79d9f6d7 100755 --- a/cgi/product_image_upload.pl +++ b/cgi/product_image_upload.pl @@ -116,7 +116,7 @@ my $extension = lc($1); $tmp_filename = get_string_id_for_lang("no_language", remote_addr() . '_' . $`); - ensure_dir_created($BASE_DIRS{CACHE_TMP}) or display_error_and_exit("Missing path", 503); + ensure_dir_created($BASE_DIRS{CACHE_TMP}) or display_error_and_exit($request_ref, "Missing path", 503); open(my $out, ">", "$BASE_DIRS{CACHE_TMP}/$tmp_filename.$extension"); while (my $chunk = <$file>) { print $out $chunk; @@ -176,7 +176,7 @@ my $interface_version = '20120622'; # Check that the image directory exists -ensure_dir_created($BASE_DIRS{PRODUCTS_IMAGES}) or display_error_and_exit("Missing path", 503); +ensure_dir_created($BASE_DIRS{PRODUCTS_IMAGES}) or display_error_and_exit($request_ref, "Missing path", 503); if ($imagefield) { diff --git a/cgi/product_jqm_multilingual.pl b/cgi/product_jqm_multilingual.pl index b256829f73670..70853b1547825 100755 --- a/cgi/product_jqm_multilingual.pl +++ b/cgi/product_jqm_multilingual.pl @@ -140,7 +140,7 @@ =head1 DESCRIPTION my @errors = (); # Store parameters for debug purposes - ensure_dir_created($BASE_DIRS{CACHE_DEBUG}) or display_error_and_exit("Missing path", 503); + ensure_dir_created($BASE_DIRS{CACHE_DEBUG}) or display_error_and_exit($request_ref, "Missing path", 503); open(my $out, ">", "$BASE_DIRS{CACHE_DEBUG}/product_jqm_multilingual." . time() . "." . $code); print $out encode_json(Vars()); close $out; diff --git a/cgi/product_multilingual.pl b/cgi/product_multilingual.pl index 0c2b7242a292d..08b119f64100c 100755 --- a/cgi/product_multilingual.pl +++ b/cgi/product_multilingual.pl @@ -67,6 +67,8 @@ use File::Copy qw(move); use Data::Dumper; +my $request_ref = ProductOpener::Display::init_request(); + # Function to display a form to add a product with a specific barcode (either typed in a field, or extracted from a barcode photo) # or without a barcode @@ -76,7 +78,7 @@ () if (($server_options{producers_platform}) and not((defined $Owner_id) and (($Owner_id =~ /^org-/) or ($User{moderator}) or $User{pro_moderator}))) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } my $html = ''; @@ -165,10 +167,9 @@ ($product_ref) return; } -my $request_ref = ProductOpener::Display::init_request(); - if ($User_id eq 'unwanted-user-french') { display_error_and_exit( + $request_ref, "Il y a des problèmes avec les modifications de produits que vous avez effectuées. Ce compte est temporairement bloqué, merci de nous contacter.", 403 ); @@ -223,7 +224,7 @@ ($product_ref) $code = process_search_image_form(\$filename); } elsif (not is_valid_code($code)) { - display_error_and_exit($Lang{invalid_barcode}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{invalid_barcode}{$lc}, 403); } my $r = Apache2::RequestUtil->request(); @@ -315,28 +316,28 @@ ($product_ref) else { # We should have a code if ((not defined $code) or ($code eq '')) { - display_error_and_exit($Lang{missing_barcode}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{missing_barcode}{$lc}, 403); } elsif (not is_valid_code($code)) { - display_error_and_exit($Lang{invalid_barcode}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{invalid_barcode}{$lc}, 403); } else { if ( ((defined $server_options{private_products}) and ($server_options{private_products})) and (not defined $Owner_id)) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } $product_id = product_id_for_owner($Owner_id, $code); $product_ref = retrieve_product_or_deleted_product($product_id, $User{moderator}); if (not defined $product_ref) { - display_error_and_exit(sprintf(lang("no_product_for_barcode"), $code), 404); + display_error_and_exit($request_ref, sprintf(lang("no_product_for_barcode"), $code), 404); } } } if (($type eq 'delete') and (not $User{moderator})) { - display_error_and_exit($Lang{error_no_permission}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{error_no_permission}{$lc}, 403); } if ($User_id eq 'unwanted-bot-id') { @@ -382,7 +383,7 @@ ($product_ref) if (not $proceed_with_edit) { - display_error_and_exit("Edit against edit rules", 403); + display_error_and_exit($request_ref, "Edit against edit rules", 403); } $log->debug("phase 1", {code => $code, type => $type}) if $log->is_debug(); diff --git a/cgi/remove_products.pl b/cgi/remove_products.pl index 603bf7c0a2bd3..279999ffae8c3 100644 --- a/cgi/remove_products.pl +++ b/cgi/remove_products.pl @@ -55,11 +55,11 @@ $template_data_ref->{action} = $action; if (not $server_options{producers_platform}) { - display_error_and_exit(lang("function_not_available"), 200); + display_error_and_exit($request_ref, lang("function_not_available"), 200); } if ((not defined $Owner_id) or ($Owner_id !~ /^(user|org)-\S+$/)) { - display_error_and_exit(lang("no_owner_defined"), 200); + display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } if ($action eq "display") { @@ -82,7 +82,7 @@ File::Copy::Recursive->import(qw( dirmove )); my $deleted_dir = $BASE_DIRS{DELETED_PRIVATE_PRODUCTS} . "/" . $Owner_id . "." . time(); - ensure_dir_created($BASE_DIRS{PRODUCTS_IMAGES}) or display_error_and_exit("Missing path", 503); + ensure_dir_created($BASE_DIRS{PRODUCTS_IMAGES}) or display_error_and_exit($request_ref, "Missing path", 503); $log->debug("Moving data to deleted dir", {owner => $Owner_id, deleted_dir => $deleted_dir}) if $log->is_debug(); diff --git a/cgi/reset_password.pl b/cgi/reset_password.pl index e318fad3624d2..1cfe1a418c451 100755 --- a/cgi/reset_password.pl +++ b/cgi/reset_password.pl @@ -59,7 +59,7 @@ my $html = ''; if (defined $User_id) { - display_error_and_exit($Lang{error_reset_already_connected}{$lc}, undef); + display_error_and_exit($request_ref, $Lang{error_reset_already_connected}{$lc}, undef); } if ($action eq 'process') { @@ -95,7 +95,7 @@ } else { $log->debug("invalid address", {type => $type}) if $log->is_debug(); - display_error_and_exit(lang("error_invalid_address"), 404); + display_error_and_exit($request_ref, lang("error_invalid_address"), 404); } if ($#errors >= 0) { @@ -173,7 +173,7 @@ } else { $log->debug("token is invalid", {userid => $userid}) if $log->is_debug(); - display_error_and_exit($Lang{error_reset_invalid_token}{$lc}, undef); + display_error_and_exit($request_ref, $Lang{error_reset_invalid_token}{$lc}, undef); } } } diff --git a/cgi/search.pl b/cgi/search.pl index a6b2357f4657a..8e7c4c3560392 100755 --- a/cgi/search.pl +++ b/cgi/search.pl @@ -46,6 +46,8 @@ use JSON::PP; use Log::Any qw($log); +my $request_ref = ProductOpener::Display::init_request(); + # Passing values to the template my $template_data_ref = {}; @@ -58,12 +60,12 @@ if (user_agent() =~ /apps-spreadsheets/) { display_error_and_exit( + $request_ref, "Automated queries using Google Spreadsheet overload the Open Food Facts server. We cannot support them. You can contact us at contact\@openfoodfacts.org to tell us about your use case, so that we can see if there is another way to support it.", 200 ); } -my $request_ref = ProductOpener::Display::init_request(); $request_ref->{search} = 1; my $action = single_param('action') || 'display'; diff --git a/cgi/user.pl b/cgi/user.pl index 09833acb428bd..5ef2df47b70f0 100644 --- a/cgi/user.pl +++ b/cgi/user.pl @@ -88,7 +88,7 @@ if ($type =~ /^edit/) { $user_ref = retrieve_user($userid); if (not defined $user_ref) { - display_error_and_exit($Lang{error_invalid_user}{$lc}, 404); + display_error_and_exit($request_ref, $Lang{error_invalid_user}{$lc}, 404); } } else { @@ -96,7 +96,7 @@ } if (($type =~ /^edit/) and ($User_id ne $userid) and not $admin) { - display_error_and_exit($Lang{error_no_permission}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{error_no_permission}{$lc}, 403); } my $debug = 0; @@ -117,11 +117,11 @@ ProductOpener::Users::check_edit_owner($user_ref, \@errors); } else { - display_error_and_exit($Lang{error_no_permission}{$lc}, 403); + display_error_and_exit($request_ref, $Lang{error_no_permission}{$lc}, 403); } } elsif ($type ne 'delete') { - ProductOpener::Users::check_user_form($type, $user_ref, \@errors); + ProductOpener::Users::check_user_form($request_ref, $type, $user_ref, \@errors); } if ($#errors >= 0) { diff --git a/cpanfile b/cpanfile index 59db71f47dfdb..b25664f9b665d 100644 --- a/cpanfile +++ b/cpanfile @@ -123,6 +123,7 @@ on 'test' => sub { requires 'Devel::Cover::Report::Codecovbash'; requires 'Test::Fake::HTTPD'; requires 'URL::Encode'; + requires 'Test::File::Contents'; requires 'FindBin'; requires 'Test::Pod'; }; diff --git a/lib/ProductOpener/APITest.pm b/lib/ProductOpener/APITest.pm index 5909397ac4903..e607a389e418c 100644 --- a/lib/ProductOpener/APITest.pm +++ b/lib/ProductOpener/APITest.pm @@ -499,7 +499,7 @@ sub execute_request ($test_ref, $ua) { # We would need to re-construct the url my $final_url = $response->request->uri; if ($url ne $final_url) { - diag("Got a redirect to " . $final_url); + diag("Warning: redirects are not supported by APITest.pm!!! Got a redirect to " . $final_url); } return $response; @@ -539,18 +539,21 @@ sub check_request_response ($test_ref, $response, $test_id, $test_dir, $expected diag("Response content: " . $response_content); } - if ((defined $test_ref->{expected_type}) and ($test_ref->{expected_type} eq 'text')) { - # Check that the text file is the same as expected (useful for checking dynamic robots.txt) + my $expected_type = $test_ref->{expected_type} // 'json'; + + if ((($expected_type eq 'text') or ($expected_type eq 'html'))) { + # Check that the file is the same as expected (useful for HTML content or dynamic robots.txt) is( compare_file_to_expected_results( - $response_content, "$expected_result_dir/$test_case.txt", + $response_content, "$expected_result_dir/$test_case.$expected_type", $update_expected_results, $test_ref ), 1, "$test_case - result" ); } - elsif (not((defined $test_ref->{expected_type}) and ($test_ref->{expected_type} eq "html"))) { + # Otherwise we expect the result is JSON + elsif ($expected_type eq 'json') { # Check that we got a JSON response @@ -565,7 +568,7 @@ sub check_request_response ($test_ref, $response, $test_id, $test_dir, $expected ); diag("Response content: " . $response_content); fail($test_case); - next; + return; }; # If the request was a setup request, we don't need to save or check the response @@ -596,6 +599,11 @@ sub check_request_response ($test_ref, $response, $test_id, $test_dir, $expected } } + # We do not check the response content for some queries (e.g OPTIONS queries), in that case expected_type is set to 'none' + elsif ($expected_type ne 'none') { + fail($test_case); + diag("Unknown expected type: $expected_type"); + } # Check if the response content matches what we expect my $must_match = $test_ref->{response_content_must_match}; diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index ee7b39f6f4a13..1f3fc0f0c0d34 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -199,6 +199,7 @@ use Devel::Size qw(size total_size); use Data::DeepAccess qw(deep_get deep_set); use Log::Log4perl; use LWP::UserAgent; +use Tie::IxHash; use Log::Any '$log', default_adapter => 'Stderr'; @@ -225,6 +226,12 @@ my $uri_finder = URI::Find->new( } ); +# Sort keys of JSON output +# $json has utf8 disabled: it encodes to Perl Unicode strings +my $json = JSON::PP->new->utf8(0)->allow_nonref->canonical; +# $json_utf8 has utf8 enabled: it encodes to UTF-8 bytes +my $json_utf8 = JSON::PP->new->utf8(1)->allow_nonref->canonical; + =head1 VARIABLES Exported variables that are available for other modules. @@ -418,11 +425,6 @@ sub process_template ($template_filename, $template_data_ref, $result_content_re $permission); }; - # select2 options generator for all entries in a taxonomy - $template_data_ref->{generate_select2_options_for_taxonomy_to_json} = sub ($tagtype) { - return generate_select2_options_for_taxonomy_to_json($lc, $tagtype); - }; - # Return a link to one taxonomy entry in the target language $template_data_ref->{canonicalize_taxonomy_tag_link} = sub ($tagtype, $tag) { return canonicalize_taxonomy_tag_link($lc, $tagtype, $tag); @@ -456,7 +458,7 @@ sub process_template ($template_filename, $template_data_ref, $result_content_re }; $template_data_ref->{encode_json} = sub ($var) { - return decode_utf8(JSON::PP->new->utf8->canonical->encode($var)); + return $json->encode($var); }; return ($tt->process($template_filename, $template_data_ref, $result_content_ref)); @@ -782,6 +784,14 @@ sub init_request ($request_ref = {}) { } } + # Set cc, lc and lcs in the request object + # Ideally, we should rely on those fields in the request object + # and remove the $lc, $cc and @lcs global variables + $request_ref->{lc} = $lc; + $request_ref->{cc} = $cc; + $request_ref->{country} = $country; + $request_ref->{lcs} = \@lcs; + # If lc is not one of the official languages of the country and if the request comes from # a bot crawler, don't index the webpage (return an empty noindex HTML page) # We also disable indexing for all subdomains that don't have the format world, cc or cc-lc @@ -848,7 +858,7 @@ sub init_request ($request_ref = {}) { "init_request - init_user error - display error page", {init_user_error => $request_ref->{init_user_error}} ) if $log->is_debug(); - display_error_and_exit($error, 403); + display_error_and_exit($request_ref, $error, 403); } } @@ -950,14 +960,6 @@ CSS } ) if $log->is_debug(); - # Set cc, lc and lcs in the request object - # Ideally, we should rely on those fields in the request object - # and remove the $lc, $cc and @lcs global variables - $request_ref->{lc} = $lc; - $request_ref->{cc} = $cc; - $request_ref->{country} = $country; - $request_ref->{lcs} = \@lcs; - return $request_ref; } @@ -1084,7 +1086,7 @@ sub display_date_iso ($t) { } } -=head2 display_error ( $error_message, $status_code ) +=head2 display_error ( $request_ref, $error_message, $status_code ) Display an error message using the site template. @@ -1092,21 +1094,30 @@ The request is not terminated by this function, it will continue to run. =cut -sub display_error ($error_message, $status_code) { +sub display_error ($request_ref, $error_message, $status_code) { + + $log->debug("display_error", + {error_message => $error_message, status_code => $status_code, request_ref => $request_ref}) + if $log->is_debug(); + + # We need to remove the canonical URL from the request so that it does not get displayed in the error page + # This is needed in particular for facet pages like /some-facet/some-spam-value-that-we-don-t-want-to-output-in-the-error-page + delete $request_ref->{canon_url}; + delete $request_ref->{canon_rel_url}; + delete $request_ref->{url}; + delete $request_ref->{current_link}; my $html = "

$error_message

"; - display_page( - { - title => lang('error'), - content_ref => \$html, - status_code => $status_code, - page_type => "error", - } - ); + $request_ref->{status_code} = $status_code; + $request_ref->{page_type} = "error"; + $request_ref->{title} = lang('error'); + $request_ref->{content_ref} = \$html; + + display_page($request_ref); return; } -=head2 display_error_and_exit ( $error_message, $status_code ) +=head2 display_error_and_exit ( $request_ref, $error_message, $status_code ) Display an error message using the site template, and terminate the request immediately. @@ -1114,9 +1125,9 @@ Any code after the call to display_error_and_exit() will not be executed. =cut -sub display_error_and_exit ($error_message, $status_code) { +sub display_error_and_exit ($request_ref, $error_message, $status_code) { - display_error($error_message, $status_code); + display_error($request_ref, $error_message, $status_code); exit(); } @@ -1638,14 +1649,18 @@ sub query_list_of_tags ($request_ref, $query_ref) { # allow sorting by tagname my $sort_by = request_param($request_ref, "sort_by") // $default_sort_by; - my $sort_ref; + my %sort = (); + + # We need a tie hash so that the keys are ordered by insertion order when passed to MongoDB + tie(%sort, 'Tie::IxHash'); + my $sort_ref = \%sort; if ($sort_by eq "tag") { - $sort_ref = {"_id" => 1}; + $sort_ref->{"_id"} = 1; } else { - $sort_ref = {"count" => -1}; - $sort_by = "count"; + $sort_ref->{"count"} = -1; + $sort_ref->{"_id"} = 1; } # groupby_tagtype @@ -2422,7 +2437,6 @@ HTML # countries map? if (keys %{$countries_map_data} > 0) { - my $json = JSON::PP->new->utf8(0); $initjs .= 'var countries_map_data=JSON.parse(' . $json->encode($json->encode($countries_map_data)) . ');' .= 'var countries_map_links=JSON.parse(' . $json->encode($json->encode($countries_map_links)) . ');' .= 'var countries_map_names=JSON.parse(' . $json->encode($json->encode($countries_map_names)) . ');' @@ -2457,11 +2471,11 @@ HTML } $initjs .= < 0) or ((scalar @markers) > 0)) { - my $json = JSON::PP->new->utf8(0); my $map_template_data_ref = { - lang => \&lang, - encode_json => sub ($obj_ref) { - return $json->encode($obj_ref); - }, wikidata => \@wikidata_objects, pointers => \@markers }; @@ -3987,7 +3996,7 @@ HTML $user_or_org_ref = retrieve_org($orgid); if (not defined $user_or_org_ref) { - display_error_and_exit(lang("error_unknown_org"), 404); + display_error_and_exit($request_ref, lang("error_unknown_org"), 404); } } elsif ($tagid =~ /\./) { @@ -4009,7 +4018,7 @@ HTML $user_or_org_ref = retrieve_user($tagid); if (not defined $user_or_org_ref) { - display_error_and_exit(lang("error_unknown_user"), 404); + display_error_and_exit($request_ref, lang("error_unknown_user"), 404); } } @@ -4269,7 +4278,7 @@ HTML ) ) { - display_error_and_exit(lang("no_products"), 404); + display_error_and_exit($request_ref, lang("no_products"), 404); } else { display_page($request_ref); @@ -4353,13 +4362,11 @@ sub display_search_results ($request_ref) { $search_api_url =~ s/\&/\?/; } - my $contributor_prefs_json = decode_utf8( - encode_json( - { - display_barcode => $User{display_barcode}, - edit_link => $User{edit_link}, - } - ) + my $contributor_prefs_json = $json->encode( + { + display_barcode => $User{display_barcode}, + edit_link => $User{edit_link}, + } ); my $preferences_text = lang("classify_products_according_to_your_preferences"); @@ -5386,16 +5393,14 @@ sub search_and_display_products ($request_ref, $query_ref, $sort_by, $limit, $pa my $products_json = '[]'; if (defined $request_ref->{structured_response}{products}) { - $products_json = decode_utf8(encode_json($request_ref->{structured_response}{products})); + $products_json = $json->encode($request_ref->{structured_response}{products}); } - my $contributor_prefs_json = decode_utf8( - encode_json( - { - display_barcode => $User{display_barcode}, - edit_link => $User{edit_link}, - } - ) + my $contributor_prefs_json = $json->encode( + { + display_barcode => $User{display_barcode}, + edit_link => $User{edit_link}, + } ); $scripts .= <{current_link_query_display} =~ s/\?action=process/\?action=display/; } - my $json = JSON::PP->new->utf8(0); my $map_template_data_ref = { - lang => \&lang, - encode_json => sub ($obj_ref) { - return $json->encode($obj_ref); - }, title => $count_string, pointers => \@pointers, current_link => $request_ref->{current_link}, @@ -7521,7 +7521,7 @@ sub display_product ($request_ref) { local $log->context->{code} = $code; if (not is_valid_code($code)) { - display_error_and_exit(lang_in_other_lc($request_lc, "invalid_barcode"), 403); + display_error_and_exit($request_ref, lang_in_other_lc($request_lc, "invalid_barcode"), 403); } my $product_id = product_id_for_owner($Owner_id, $code); @@ -7570,7 +7570,7 @@ JS } if (not defined $product_ref) { - display_error_and_exit(sprintf(lang("no_product_for_barcode"), $code), 404); + display_error_and_exit($request_ref, sprintf(lang("no_product_for_barcode"), $code), 404); } $title = product_name_brand_quantity($product_ref); @@ -8258,7 +8258,7 @@ HTML compute_attributes($product_ref, $lc, $cc, $attributes_options_ref); my $product_attribute_groups_json - = decode_utf8(encode_json({"attribute_groups" => $product_ref->{"attribute_groups_" . $lc}})); + = $json->encode({"attribute_groups" => $product_ref->{"attribute_groups_" . $lc}}); my $preferences_text = lang("classify_products_according_to_your_preferences"); $scripts .= <{structured_response}); - # Sort keys of the JSON output - my $json = JSON::PP->new->allow_nonref->canonical; - my $data = $json->utf8->encode($request_ref->{structured_response}); + # We need to output binary UTF8 encoded JSON + my $data = $json_utf8->encode($request_ref->{structured_response}); my $jsonp = undef; @@ -11384,54 +11382,4 @@ sub data_to_display_image ($product_ref, $imagetype, $target_lc) { return $image_ref; } -=head2 generate_select2_options_for_taxonomy ($target_lc, $tagtype) - -Generates an array of taxonomy entries in a specific language, to be used as options -in a select2 input. - -See https://select2.org/data-sources/arrays - -=head3 Arguments - -=head4 Language code $target_lc - -=head4 Taxonomy $tagtype - -=head3 Return values - -- Reference to an array of options - -=cut - -sub generate_select2_options_for_taxonomy ($target_lc, $tagtype) { - - my @entries = (); - - # all tags can be retrieved from the $translations_to hash - foreach my $canon_tagid (keys %{$translations_to{$tagtype}}) { - # just_synonyms are not real entries - next if defined $just_synonyms{$tagtype}{$canon_tagid}; - - push @entries, display_taxonomy_tag($target_lc, $tagtype, $canon_tagid); - } - - my @options = (); - - foreach my $entry (sort @entries) { - push @options, - { - id => $entry, - text => $entry, - }; - } - - return \@options; -} - -sub generate_select2_options_for_taxonomy_to_json ($target_lc, $tagtype) { - - return decode_utf8( - JSON::PP->new->utf8->canonical->encode(generate_select2_options_for_taxonomy($target_lc, $tagtype))); -} - 1; diff --git a/lib/ProductOpener/Tags.pm b/lib/ProductOpener/Tags.pm index 6f5c4745641d4..8b5ea4783bdc6 100644 --- a/lib/ProductOpener/Tags.pm +++ b/lib/ProductOpener/Tags.pm @@ -1058,6 +1058,7 @@ sub get_file_from_cache ($source, $target) { # Change this version string if you want to force the taxonomies to be rebuilt # e.g. if the taxonomy building algorithm or configuration has changed # This needs to be done also when the unaccenting parameters for languages set in Config.pm are changed + my $BUILD_TAGS_VERSION = "20240403 - fix issue with additives.properties.txt not loaded + circular_parent check + moved canonicalization of properties to linter"; diff --git a/lib/ProductOpener/Test.pm b/lib/ProductOpener/Test.pm index 854b504a4441c..2852921a8b399 100644 --- a/lib/ProductOpener/Test.pm +++ b/lib/ProductOpener/Test.pm @@ -45,6 +45,7 @@ BEGIN { &normalize_org_for_test_comparison &normalize_product_for_test_comparison &normalize_products_for_test_comparison + &normalize_html_for_test_comparison &sort_products_for_test_comparison &normalize_user_for_test_comparison &remove_all_products @@ -77,6 +78,7 @@ use File::Path qw/make_path remove_tree/; use File::Copy; use Path::Tiny qw/path/; use Scalar::Util qw(looks_like_number); +use Test::File::Contents qw/files_eq_or_diff/; use Log::Any qw($log); @@ -388,7 +390,7 @@ sub compare_to_expected_results ($object_ref, $expected_results_file, $update_ex =head2 compare_file_to_expected_results($content_str, $expected_results_file, $update_expected_results, $test_ref = undef) { -Compare an string (e.g. text or HTML file) to expected results. +Compare a file (e.g. text or HTML file) to expected results. The expected result is stored as a plain text file. @@ -418,6 +420,11 @@ sub compare_file_to_expected_results ($content_str, $expected_results_file, $upd $desc = $test_ref->{desc} // $test_ref->{id}; } + # Normalize html + if ($expected_results_file =~ /\.html$/) { + normalize_html_for_test_comparison(\$content_str); + } + if ($update_expected_results) { open(my $result, ">:encoding(UTF-8)", $expected_results_file) or confess("Could not create $expected_results_file: $!"); @@ -434,7 +441,15 @@ sub compare_file_to_expected_results ($content_str, $expected_results_file, $upd $title = $test_ref->{desc} // $test_ref->{test_case} // $test_ref->{id}; $title = undef unless $title; } - is($content_str, $expected_result, $title); + + my $results_file = $expected_results_file . ".test"; + open(my $result, ">:encoding(UTF-8)", $results_file) + or confess("Could not create $results_file: $!"); + print $result $content_str; + close($result); + + files_eq_or_diff($expected_results_file, $results_file, $title); + unlink($results_file); } else { fail("could not load $expected_results_file"); @@ -807,6 +822,31 @@ sub normalize_org_for_test_comparison ($org_ref) { return; } +=head2 normalize_html_for_test_comparison ($html_ref) + +Normalize the HTML of a web page to be able to compare them across tests runs. + +We remove time dependent fields. + +=head3 Arguments + +=head4 product_ref - Hash ref containing product information + +=cut + +sub normalize_html_for_test_comparison ($html_ref) { + + # Remove timestamps + # + + $$html_ref =~ s/\?v=\d+/\?v=--ignore--/g; + + # + $$html_ref =~ s/