diff --git a/lib/ProductOpener/Export.pm b/lib/ProductOpener/Export.pm index 69fc17982bd10..fa40308354c1f 100644 --- a/lib/ProductOpener/Export.pm +++ b/lib/ProductOpener/Export.pm @@ -104,6 +104,7 @@ use ProductOpener::Ecoscore qw/localize_ecoscore/; use Text::CSV; use Excel::Writer::XLSX; use Data::DeepAccess qw(deep_get deep_exists); +use Apache2::RequestRec; =head1 FUNCTIONS @@ -392,7 +393,6 @@ sub export_csv ($args_ref) { # Send HTTP headers, unless search_and_export_products() is called from a script if ($send_http_headers) { - require Apache2::RequestRec; my $r = Apache2::RequestUtil->request(); $r->headers_out->set("Content-type" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); $r->headers_out->set("Content-disposition" => "attachment;filename=$filename"); diff --git a/lib/ProductOpener/GS1.pm b/lib/ProductOpener/GS1.pm index 17d3c9e8676ef..eba83b03a6f63 100644 --- a/lib/ProductOpener/GS1.pm +++ b/lib/ProductOpener/GS1.pm @@ -1721,7 +1721,7 @@ sub convert_gs1_json_message_to_off_products_csv ($json_ref, $products_ref, $mes # assign the lang and lc fields if (defined $product_ref->{languages}) { - my @sorted_languages = sort ({$product_ref->{languages}{$b} <=> $product_ref->{languages}{$a}} + my @sorted_languages = sort({$product_ref->{languages}{$b} <=> $product_ref->{languages}{$a}} keys %{$product_ref->{languages}}); my $top_language = $sorted_languages[0]; $product_ref->{lc} = $top_language; diff --git a/lib/ProductOpener/Images.pm b/lib/ProductOpener/Images.pm index f85450998e4bc..ac204feb3ae1a 100644 --- a/lib/ProductOpener/Images.pm +++ b/lib/ProductOpener/Images.pm @@ -142,6 +142,7 @@ use Encode; use JSON::PP; use MIME::Base64; use LWP::UserAgent; +use File::Copy; =head1 SUPPORTED IMAGE TYPES @@ -1113,7 +1114,6 @@ sub process_image_move ($user_id, $code, $imgids, $move_to, $ownerid) { -e "$data_root/deleted.images" or mkdir("$data_root/deleted.images", 0755); - require File::Copy; File::Copy->import(qw( move )); $log->info( diff --git a/lib/ProductOpener/Import.pm b/lib/ProductOpener/Import.pm index 9c33353c3d350..ba3e4d7efb290 100644 --- a/lib/ProductOpener/Import.pm +++ b/lib/ProductOpener/Import.pm @@ -109,6 +109,7 @@ use Text::CSV; use DateTime::Format::ISO8601; use URI; use Digest::MD5 qw(md5_hex); +use LWP::UserAgent; # private function to import images from dir # args: @@ -231,8 +232,6 @@ sub import_images_from_dir ($image_dir, $stats) { # download image at given url parameter sub download_image ($image_url) { - require LWP::UserAgent; - my $ua = LWP::UserAgent->new(timeout => 10); # Some platforms such as CloudFlare block the default LWP user agent. diff --git a/lib/ProductOpener/LoadData.pm b/lib/ProductOpener/LoadData.pm new file mode 100644 index 0000000000000..f84af57a72f0c --- /dev/null +++ b/lib/ProductOpener/LoadData.pm @@ -0,0 +1,91 @@ +# This file is part of Product Opener. +# +# Product Opener +# Copyright (C) 2011-2021 Association Open Food Facts +# Contact: contact@openfoodfacts.org +# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France +# +# Product Opener is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +=head1 NAME + +ProductOpener::LoadData - Load and initialize data + +=head1 DESCRIPTION + +This module provides a load_data() module that loads and initializes data needed by Product Opener. + +=cut + +package ProductOpener::LoadData; + +use ProductOpener::PerlStandards; +use Exporter qw< import >; + +use Log::Any qw($log); + +BEGIN { + use vars qw(@ISA @EXPORT_OK %EXPORT_TAGS); + @EXPORT_OK = qw( + + &load_data + + ); # symbols to export on request + %EXPORT_TAGS = (all => [@EXPORT_OK]); +} + +use vars @EXPORT_OK; + +use ProductOpener::Config qw/:all/; +use ProductOpener::Tags qw/:all/; +use ProductOpener::PackagerCodes qw/:all/; +use ProductOpener::Packaging qw/:all/; +use ProductOpener::ForestFootprint qw/:all/; +use ProductOpener::Ecoscore qw(:all); +use ProductOpener::MainCountries qw(:all); + +=head1 FUNCTIONS + +=head2 load_data() + +loads and initializes data needed by Product Opener. + +It needs to be called once at startup: +- in lib/startup_apache2.pl for Apache +- in script files + +=cut + +sub load_data() { + + $log->debug("loading data - start") if $log->is_debug(); + + init_emb_codes(); + init_packager_codes(); + init_geocode_addresses(); + init_packaging_taxonomies_regexps(); + load_scans_data(); + + if ((defined $options{product_type}) and ($options{product_type} eq "food")) { + load_agribalyse_data(); + load_ecoscore_data(); + load_forest_footprint_data(); + } + + $log->debug("loading data - done") if $log->is_debug(); + + return; +} + +1; diff --git a/lib/ProductOpener/PackagerCodes.pm b/lib/ProductOpener/PackagerCodes.pm index 8cb0f4b480eb1..4447e016b5ae8 100644 --- a/lib/ProductOpener/PackagerCodes.pm +++ b/lib/ProductOpener/PackagerCodes.pm @@ -256,10 +256,4 @@ sub init_geocode_addresses() { return; } -# Slow, so only run these when actually executing, not just checking syntax. See also startup_apache2.pl. -INIT { - init_packager_codes(); - init_geocode_addresses(); -} - 1; diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm index 644d35a64180d..498a780b13ee4 100644 --- a/lib/ProductOpener/Products.pm +++ b/lib/ProductOpener/Products.pm @@ -141,6 +141,8 @@ use Data::DeepAccess qw(deep_get); use LWP::UserAgent; use Storable qw(dclone); +use File::Copy::Recursive; +use ProductOpener::GeoIP; use Algorithm::CheckDigits; my $ean_check = CheckDigits('ean'); @@ -642,8 +644,6 @@ sub init_product ($userid, $orgid, $code, $countryid) { my $country; if (((not defined $countryid) or ($countryid eq "en:world")) and (remote_addr() ne "127.0.0.1")) { - - require ProductOpener::GeoIP; $country = ProductOpener::GeoIP::get_country_for_ip(remote_addr()); } elsif (defined $countryid) { @@ -1047,7 +1047,6 @@ sub store_product ($user_id, $product_ref, $comment) { # # use File::Copy; - require File::Copy::Recursive; File::Copy::Recursive->import(qw( dirmove )); $log->debug("moving product data", @@ -2926,7 +2925,6 @@ sub process_product_edit_rules ($product_ref) { $emoji = ":pear:"; } - require LWP::UserAgent; my $ua = LWP::UserAgent->new; my $server_endpoint = "https://hooks.slack.com/services/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"; diff --git a/lib/startup_apache2.pl b/lib/startup_apache2.pl index 88349ff881340..7768b69cae758 100755 --- a/lib/startup_apache2.pl +++ b/lib/startup_apache2.pl @@ -24,8 +24,8 @@ # (instead of when each httpd child starts) # see http://apache.perl.org/docs/1.0/guide/performance.html#Code_Profiling_Techniques # -use utf8; -use Modern::Perl '2017'; + +use ProductOpener::PerlStandards; use Carp (); @@ -61,7 +61,6 @@ Log::Any::Adapter->set('Log4perl'); # Send all logs to Log::Log4perl use ProductOpener::Lang qw/:all/; - use ProductOpener::Store qw/:all/; use ProductOpener::Display qw/:all/; use ProductOpener::Products qw/:all/; @@ -72,6 +71,8 @@ use ProductOpener::URL qw/:all/; use ProductOpener::Version qw/:all/; use ProductOpener::DataQuality qw/:all/; +use ProductOpener::DataQualityCommon qw/:all/; +use ProductOpener::DataQualityFood qw/:all/; use ProductOpener::Packaging qw/:all/; use ProductOpener::ForestFootprint qw/:all/; use ProductOpener::Nutriscore qw(:all); @@ -83,6 +84,25 @@ use ProductOpener::Recipes qw(:all); use ProductOpener::MainCountries qw/:all/; use ProductOpener::PackagerCodes qw/:all/; +#use ProductOpener::API qw/:all/; +#use ProductOpener::APITest qw/:all/; +#use ProductOpener::APIProductRead qw/:all/; +#use ProductOpener::APIProductWrite qw/:all/; +#use ProductOpener::Routing qw/:all/; +use ProductOpener::Mail qw/:all/; +use ProductOpener::Export qw/:all/; +use ProductOpener::Import qw/:all/; +use ProductOpener::ImportConvert qw/:all/; +use ProductOpener::Numbers qw/:all/; +use ProductOpener::Producers qw/:all/; +use ProductOpener::ProducersFood qw/:all/; +use ProductOpener::GeoIP qw/:all/; +use ProductOpener::GS1 qw/:all/; +use ProductOpener::Redis qw/:all/; +use ProductOpener::FoodGroups qw/:all/; +use ProductOpener::Events qw/:all/; +use ProductOpener::Data qw/:all/; +use ProductOpener::LoadData qw/:all/; use Apache2::Const -compile => qw(OK); use Apache2::Connection (); @@ -120,18 +140,7 @@ sub get_remote_proxy_address { print {*STDERR} $log or Carp::croak('Unable to write to *STDERR'); # load large data files into mod_perl memory -init_emb_codes(); -init_packager_codes(); -init_geocode_addresses(); -init_packaging_taxonomies_regexps(); - -load_scans_data(); - -if ((defined $options{product_type}) and ($options{product_type} eq "food")) { - load_agribalyse_data(); - load_ecoscore_data(); - load_forest_footprint_data(); -} +load_data(); # This startup script is run as root, it will create the $data_root/tmp directory # if it does not exist, as well as sub-directories for the Template module diff --git a/scripts/import_csv_file.pl b/scripts/import_csv_file.pl index 1d44fe8cdb512..c446c42c5b407 100755 --- a/scripts/import_csv_file.pl +++ b/scripts/import_csv_file.pl @@ -41,6 +41,7 @@ use ProductOpener::Packaging qw/:all/; use ProductOpener::ForestFootprint qw/:all/; use ProductOpener::PackagerCodes qw/:all/; +use ProductOpener::LoadData qw/:all/; use URI::Escape::XS; use Storable qw/dclone/; @@ -185,16 +186,7 @@ $missing_arg and exit(); -init_emb_codes(); -init_packager_codes(); -init_geocode_addresses(); -init_packaging_taxonomies_regexps(); - -if ((defined $options{product_type}) and ($options{product_type} eq "food")) { - load_agribalyse_data(); - load_ecoscore_data(); - load_forest_footprint_data(); -} +load_data(); my $args_ref = { user_id => $user_id, diff --git a/scripts/minion.pl b/scripts/minion.pl deleted file mode 100755 index af4ac58d1367e..0000000000000 --- a/scripts/minion.pl +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/perl -w - -# This file is part of Product Opener. -# -# Product Opener -# Copyright (C) 2011-2020 Association Open Food Facts -# Contact: contact@openfoodfacts.org -# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France -# -# Product Opener is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -use Modern::Perl '2017'; -use utf8; - -use ProductOpener::Config qw/:all/; - -use Mojolicious::Lite; - -use Minion; - -# Minion backend - -if (not defined $server_options{minion_backend}) { - - die("No Minion backend configured in lib/ProductOpener/Config2.pm\n"); -} - -plugin Minion => $server_options{minion_backend}; - -app->start; diff --git a/scripts/minion_export_test.pl b/scripts/minion_export_test.pl deleted file mode 100755 index 38f2f76c9acb4..0000000000000 --- a/scripts/minion_export_test.pl +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/perl -w - -# This file is part of Product Opener. -# -# Product Opener -# Copyright (C) 2011-2020 Association Open Food Facts -# Contact: contact@openfoodfacts.org -# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France -# -# Product Opener is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -use Modern::Perl '2017'; -use utf8; - -use ProductOpener::Config qw/:all/; -use ProductOpener::Producers qw/:all/; - -use Minion; - -$minion->add_task(export_csv_file => \&ProductOpener::Producers::export_csv_file_task); - -print STDERR "Perform 1 job in current process\n"; - -# Perform one job manually in this process -my $worker = $minion->repair->worker->register; - -my $job = $worker->dequeue(0 => {queues => [$server_options{minion_local_queue}]}); -if (defined $job) { - if (my $err = $job->execute) {print STDERR "Error: $err\n"; $job->fail($err);} - else {print STDERR "Done\n"; $job->finish} -} -else { - print STDERR "no job found in queue " . $server_options{minion_local_queue} . "\n"; -} - -$worker->unregister; - diff --git a/scripts/minion_import.pl b/scripts/minion_import.pl deleted file mode 100755 index 98a15c4a7345e..0000000000000 --- a/scripts/minion_import.pl +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/perl -w - -# This file is part of Product Opener. -# -# Product Opener -# Copyright (C) 2011-2020 Association Open Food Facts -# Contact: contact@openfoodfacts.org -# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France -# -# Product Opener is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -use Modern::Perl '2017'; -use utf8; - -use ProductOpener::Config qw/:all/; -use ProductOpener::Store qw/:all/; -use ProductOpener::Index qw/:all/; -use ProductOpener::Display qw/:all/; -use ProductOpener::Tags qw/:all/; -use ProductOpener::Users qw/:all/; -use ProductOpener::Images qw/:all/; -use ProductOpener::Lang qw/:all/; -use ProductOpener::Mail qw/:all/; -use ProductOpener::Products qw/:all/; -use ProductOpener::Producers qw/:all/; -use ProductOpener::Food qw/:all/; -use ProductOpener::Ingredients qw/:all/; -use ProductOpener::Images qw/:all/; - -use URI::Escape::XS; -use Storable qw/dclone/; -use Encode; -use JSON::PP; -use Time::Local; -use Data::Dumper; -use Text::CSV; -use Getopt::Long; - -use Mojolicious::Lite; -use Minion; - -use Log::Any::Adapter 'TAP', filter => "none"; - -# Minion backend - -if (not defined $server_options{minion_backend}) { - - die("No Minion backend configured in lib/ProductOpener/Config2.pm\n"); -} - -plugin Minion => $server_options{minion_backend}; - -# app->minion->add_task(import_csv_file => &import_csv_file_task); - -app->minion->add_task(import_csv_file => \&ProductOpener::Producers::import_csv_file_task); - -app->start; diff --git a/scripts/minion_import_test.pl b/scripts/minion_import_test.pl deleted file mode 100755 index 3143364505e79..0000000000000 --- a/scripts/minion_import_test.pl +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/perl -w - -# This file is part of Product Opener. -# -# Product Opener -# Copyright (C) 2011-2020 Association Open Food Facts -# Contact: contact@openfoodfacts.org -# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France -# -# Product Opener is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -use Modern::Perl '2017'; -use utf8; - -use ProductOpener::Config qw/:all/; -use ProductOpener::Producers qw/:all/; - -use Minion; - -$minion->add_task(import_csv_file => \&ProductOpener::Producers::import_csv_file_task); - -print STDERR "Perform 1 job in current process\n"; - -# Perform one job manually in this process -my $worker = $minion->repair->worker->register; - -my $job = $worker->dequeue(0 => {queues => [$server_options{minion_local_queue}]}); -if (my $err = $job->execute) {print STDERR "Error: $err\n"; $job->fail($err);} -else {print STDERR "Done\n"; $job->finish} -$worker->unregister; - diff --git a/scripts/minion_producers.pl b/scripts/minion_producers.pl index 58aa605e58649..88d0b0f3d6eda 100755 --- a/scripts/minion_producers.pl +++ b/scripts/minion_producers.pl @@ -33,6 +33,7 @@ use ProductOpener::ForestFootprint qw/:all/; use ProductOpener::MainCountries qw/:all/; use ProductOpener::PackagerCodes qw/:all/; +use ProductOpener::LoadData qw/:all/; use Log::Any qw($log); use Log::Log4perl; @@ -48,19 +49,7 @@ $log->info("starting minion producers workers", {minion_backend => $server_options{minion_backend}}) if $log->is_info(); -# load large data files into mod_perl memory -init_emb_codes(); -init_packager_codes(); -init_geocode_addresses(); -init_packaging_taxonomies_regexps(); - -load_scans_data(); - -if ((defined $options{product_type}) and ($options{product_type} eq "food")) { - load_agribalyse_data(); - load_ecoscore_data(); - load_forest_footprint_data(); -} +load_data(); if (not defined $server_options{minion_backend}) { diff --git a/scripts/minion_producers_test.pl b/scripts/minion_producers_test.pl deleted file mode 100755 index 2a06aeca2134b..0000000000000 --- a/scripts/minion_producers_test.pl +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/perl -w - -# This file is part of Product Opener. -# -# Product Opener -# Copyright (C) 2011-2020 Association Open Food Facts -# Contact: contact@openfoodfacts.org -# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France -# -# Product Opener is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -use Modern::Perl '2017'; -use utf8; - -use ProductOpener::Config qw/:all/; -use ProductOpener::Producers qw/:all/; - -use Minion; - -$minion->add_task(import_csv_file => \&ProductOpener::Producers::import_csv_file_task); -$minion->add_task(export_csv_file => \&ProductOpener::Producers::export_csv_file_task); -$minion->add_task(import_products_categories_from_public_database => - \&ProductOpener::Producers::import_products_categories_from_public_database_task); -$minion->add_task( - update_export_status_for_csv_file => \&ProductOpener::Producers::update_export_status_for_csv_file_task); - -print STDERR "Perform 1 job in current process\n"; - -# Perform one job manually in this process -my $worker = $minion->repair->worker->register; - -my $job = $worker->dequeue(0 => {queues => [$server_options{minion_local_queue}]}); -if (my $err = $job->execute) {print STDERR "Error: $err\n"; $job->fail($err);} -else {print STDERR "Done\n"; $job->finish} -$worker->unregister; - diff --git a/scripts/update_all_products.pl b/scripts/update_all_products.pl old mode 100644 new mode 100755 index cb3344ae3ed21..30b972c773d1d --- a/scripts/update_all_products.pl +++ b/scripts/update_all_products.pl @@ -75,6 +75,7 @@ use ProductOpener::ForestFootprint qw(:all); use ProductOpener::MainCountries qw(:all); use ProductOpener::PackagerCodes qw/:all/; +use ProductOpener::LoadData qw/:all/; use CGI qw/:cgi :form escapeHTML/; use URI::Escape::XS; @@ -194,9 +195,6 @@ @fields_to_update = split(/,/, join(',', @fields_to_update)); -use Data::Dumper; - -# simple procedural interface print Dumper(\@fields_to_update); if ((defined $team) and ($team ne "")) { @@ -265,22 +263,6 @@ die("Missing fields to update or --count option:\n$usage"); } -if ($compute_ecoscore) { - - init_packaging_taxonomies_regexps(); - load_agribalyse_data(); - load_ecoscore_data(); -} - -if ($compute_forest_footprint) { - - load_forest_footprint_data(); -} - -if ($compute_main_countries) { - load_scans_data(); -} - # Make sure we have a user id and we will use a new .sto file for all edits that change values entered by users if ((not defined $User_id) and (($fix_serving_size_mg_to_ml) or ($fix_missing_lc))) { die( @@ -288,6 +270,8 @@ ); } +load_data(); + # Get a list of all products not yet updated # Use query filters entered using --query categories_tags=en:plant-milks diff --git a/tests/update_tests_results.sh b/tests/update_tests_results.sh old mode 100644 new mode 100755