refactor: new load_data() function to be used in startup_apache2.pl a…

…nd batch scripts (#7679)

* remove warning - sort (...) interpreted as function

* load modules at startup

* refactor: load_data() function

* update tests

* lint + chmod

lib/ProductOpener/Export.pm

@@ -104,6 +104,7 @@ use ProductOpener::Ecoscore qw/localize_ecoscore/;
 use Text::CSV;
 use Excel::Writer::XLSX;
 use Data::DeepAccess qw(deep_get deep_exists);
+use Apache2::RequestRec;
 
 =head1 FUNCTIONS
 
@@ -392,7 +393,6 @@ sub export_csv ($args_ref) {
 
 		# Send HTTP headers, unless search_and_export_products() is called from a script
 		if ($send_http_headers) {
-			require Apache2::RequestRec;
 			my $r = Apache2::RequestUtil->request();
 			$r->headers_out->set("Content-type" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
 			$r->headers_out->set("Content-disposition" => "attachment;filename=$filename");

lib/ProductOpener/GS1.pm

@@ -1721,7 +1721,7 @@ sub convert_gs1_json_message_to_off_products_csv ($json_ref, $products_ref, $mes
 
 	# assign the lang and lc fields
 	if (defined $product_ref->{languages}) {
-		my @sorted_languages = sort ({$product_ref->{languages}{$b} <=> $product_ref->{languages}{$a}}
+		my @sorted_languages = sort({$product_ref->{languages}{$b} <=> $product_ref->{languages}{$a}}
 			keys %{$product_ref->{languages}});
 		my $top_language = $sorted_languages[0];
 		$product_ref->{lc} = $top_language;

lib/ProductOpener/Images.pm

@@ -142,6 +142,7 @@ use Encode;
 use JSON::PP;
 use MIME::Base64;
 use LWP::UserAgent;
+use File::Copy;
 
 =head1 SUPPORTED IMAGE TYPES
 
@@ -1113,7 +1114,6 @@ sub process_image_move ($user_id, $code, $imgids, $move_to, $ownerid) {
 
 				-e "$data_root/deleted.images" or mkdir("$data_root/deleted.images", 0755);
 
-				require File::Copy;
 				File::Copy->import(qw( move ));
 
 				$log->info(

lib/ProductOpener/Import.pm

@@ -109,6 +109,7 @@ use Text::CSV;
 use DateTime::Format::ISO8601;
 use URI;
 use Digest::MD5 qw(md5_hex);
+use LWP::UserAgent;
 
 # private function to import images from dir
 # args:
@@ -231,8 +232,6 @@ sub import_images_from_dir ($image_dir, $stats) {
 # download image at given url parameter
 sub download_image ($image_url) {
 
-	require LWP::UserAgent;
-
 	my $ua = LWP::UserAgent->new(timeout => 10);
 
 	# Some platforms such as CloudFlare block the default LWP user agent.

lib/ProductOpener/LoadData.pm

@@ -0,0 +1,91 @@
+# This file is part of Product Opener.
+#
+# Product Opener
+# Copyright (C) 2011-2021 Association Open Food Facts
+# Contact: contact@openfoodfacts.org
+# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France
+#
+# Product Opener is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+=head1 NAME
+
+ProductOpener::LoadData - Load and initialize data
+
+=head1 DESCRIPTION
+
+This module provides a load_data() module that loads and initializes data needed by Product Opener.
+
+=cut
+
+package ProductOpener::LoadData;
+
+use ProductOpener::PerlStandards;
+use Exporter qw< import >;
+
+use Log::Any qw($log);
+
+BEGIN {
+	use vars qw(@ISA @EXPORT_OK %EXPORT_TAGS);
+	@EXPORT_OK = qw(
+
+		&load_data
+
+	);    # symbols to export on request
+	%EXPORT_TAGS = (all => [@EXPORT_OK]);
+}
+
+use vars @EXPORT_OK;
+
+use ProductOpener::Config qw/:all/;
+use ProductOpener::Tags qw/:all/;
+use ProductOpener::PackagerCodes qw/:all/;
+use ProductOpener::Packaging qw/:all/;
+use ProductOpener::ForestFootprint qw/:all/;
+use ProductOpener::Ecoscore qw(:all);
+use ProductOpener::MainCountries qw(:all);
+
+=head1 FUNCTIONS
+
+=head2 load_data()
+
+loads and initializes data needed by Product Opener.
+
+It needs to be called once at startup:
+- in lib/startup_apache2.pl for Apache
+- in script files
+
+=cut
+
+sub load_data() {
+
+	$log->debug("loading data - start") if $log->is_debug();
+
+	init_emb_codes();
+	init_packager_codes();
+	init_geocode_addresses();
+	init_packaging_taxonomies_regexps();
+	load_scans_data();
+
+	if ((defined $options{product_type}) and ($options{product_type} eq "food")) {
+		load_agribalyse_data();
+		load_ecoscore_data();
+		load_forest_footprint_data();
+	}
+
+	$log->debug("loading data - done") if $log->is_debug();
+
+	return;
+}
+
+1;

lib/ProductOpener/PackagerCodes.pm

@@ -256,10 +256,4 @@ sub init_geocode_addresses() {
 	return;
 }
 
-# Slow, so only run these when actually executing, not just checking syntax. See also startup_apache2.pl.
-INIT {
-	init_packager_codes();
-	init_geocode_addresses();
-}
-
 1;

lib/ProductOpener/Products.pm

@@ -141,6 +141,8 @@ use Data::DeepAccess qw(deep_get);
 
 use LWP::UserAgent;
 use Storable qw(dclone);
+use File::Copy::Recursive;
+use ProductOpener::GeoIP;
 
 use Algorithm::CheckDigits;
 my $ean_check = CheckDigits('ean');
@@ -642,8 +644,6 @@ sub init_product ($userid, $orgid, $code, $countryid) {
 	my $country;
 
 	if (((not defined $countryid) or ($countryid eq "en:world")) and (remote_addr() ne "127.0.0.1")) {
-
-		require ProductOpener::GeoIP;
 		$country = ProductOpener::GeoIP::get_country_for_ip(remote_addr());
 	}
 	elsif (defined $countryid) {
@@ -1047,7 +1047,6 @@ sub store_product ($user_id, $product_ref, $comment) {
 			#
 			# use File::Copy;
 
-			require File::Copy::Recursive;
 			File::Copy::Recursive->import(qw( dirmove ));
 
 			$log->debug("moving product data",
@@ -2926,7 +2925,6 @@ sub process_product_edit_rules ($product_ref) {
 										$emoji = ":pear:";
 									}
 
-									require LWP::UserAgent;
 									my $ua = LWP::UserAgent->new;
 									my $server_endpoint
 										= "https://hooks.slack.com/services/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";

lib/startup_apache2.pl

@@ -24,8 +24,8 @@
 # (instead of when each httpd child starts)
 # see http://apache.perl.org/docs/1.0/guide/performance.html#Code_Profiling_Techniques
 #
-use utf8;
-use Modern::Perl '2017';
+
+use ProductOpener::PerlStandards;
 
 use Carp ();
 
@@ -61,7 +61,6 @@
 Log::Any::Adapter->set('Log4perl');    # Send all logs to Log::Log4perl
 
 use ProductOpener::Lang qw/:all/;
-
 use ProductOpener::Store qw/:all/;
 use ProductOpener::Display qw/:all/;
 use ProductOpener::Products qw/:all/;
@@ -72,6 +71,8 @@
 use ProductOpener::URL qw/:all/;
 use ProductOpener::Version qw/:all/;
 use ProductOpener::DataQuality qw/:all/;
+use ProductOpener::DataQualityCommon qw/:all/;
+use ProductOpener::DataQualityFood qw/:all/;
 use ProductOpener::Packaging qw/:all/;
 use ProductOpener::ForestFootprint qw/:all/;
 use ProductOpener::Nutriscore qw(:all);
@@ -83,6 +84,25 @@
 use ProductOpener::Recipes qw(:all);
 use ProductOpener::MainCountries qw/:all/;
 use ProductOpener::PackagerCodes qw/:all/;
+#use ProductOpener::API qw/:all/;
+#use ProductOpener::APITest qw/:all/;
+#use ProductOpener::APIProductRead qw/:all/;
+#use ProductOpener::APIProductWrite qw/:all/;
+#use ProductOpener::Routing qw/:all/;
+use ProductOpener::Mail qw/:all/;
+use ProductOpener::Export qw/:all/;
+use ProductOpener::Import qw/:all/;
+use ProductOpener::ImportConvert qw/:all/;
+use ProductOpener::Numbers qw/:all/;
+use ProductOpener::Producers qw/:all/;
+use ProductOpener::ProducersFood qw/:all/;
+use ProductOpener::GeoIP qw/:all/;
+use ProductOpener::GS1 qw/:all/;
+use ProductOpener::Redis qw/:all/;
+use ProductOpener::FoodGroups qw/:all/;
+use ProductOpener::Events qw/:all/;
+use ProductOpener::Data qw/:all/;
+use ProductOpener::LoadData qw/:all/;
 
 use Apache2::Const -compile => qw(OK);
 use Apache2::Connection ();
@@ -120,18 +140,7 @@ sub get_remote_proxy_address {
 print {*STDERR} $log or Carp::croak('Unable to write to *STDERR');
 
 # load large data files into mod_perl memory
-init_emb_codes();
-init_packager_codes();
-init_geocode_addresses();
-init_packaging_taxonomies_regexps();
-
-load_scans_data();
-
-if ((defined $options{product_type}) and ($options{product_type} eq "food")) {
-	load_agribalyse_data();
-	load_ecoscore_data();
-	load_forest_footprint_data();
-}
+load_data();
 
 # This startup script is run as root, it will create the $data_root/tmp directory
 # if it does not exist, as well as sub-directories for the Template module

scripts/import_csv_file.pl

@@ -41,6 +41,7 @@
 use ProductOpener::Packaging qw/:all/;
 use ProductOpener::ForestFootprint qw/:all/;
 use ProductOpener::PackagerCodes qw/:all/;
+use ProductOpener::LoadData qw/:all/;
 
 use URI::Escape::XS;
 use Storable qw/dclone/;
@@ -185,16 +186,7 @@
 
 $missing_arg and exit();
 
-init_emb_codes();
-init_packager_codes();
-init_geocode_addresses();
-init_packaging_taxonomies_regexps();
-
-if ((defined $options{product_type}) and ($options{product_type} eq "food")) {
-	load_agribalyse_data();
-	load_ecoscore_data();
-	load_forest_footprint_data();
-}
+load_data();
 
 my $args_ref = {
 	user_id => $user_id,