fix: Add tests for Carrefour France import, + solve serving_size bug (#…

…6476)

* tests for serving size bug

* fix incorrect serving size import #6473

* fix incorrect serving size import #6473

* fix incorrect serving size import #6473

* put carrefour import code in a module + tests

* export multiple languages in sorted order

* fix issue with random language order, fixes #2973

* fix issue with random language order, fixes #2973

* update tests

lib/ProductOpener/Import.pm

@@ -1084,6 +1084,8 @@ sub import_csv_file($) {
 						# Skip data that we have already imported before (even if it has been changed)
 						# But do import the field "obsolete"
 						elsif (($field ne "obsolete") and (defined $product_ref->{$field . "_imported"}) and ($product_ref->{$field . "_imported"} eq $imported_product_ref->{$field})) {
+							# we had a bug that caused serving_size to be set to "serving", this value should be overridden
+							next if (($field eq "serving_size") and ($product_ref->{"serving_size"} eq "serving"));
 							$log->debug("skipping field that was already imported", { field => $field, imported_value => $imported_product_ref->{$field}, current_value => $product_ref->{$field} }) if $log->is_debug();
 							next;
 						}
@@ -1771,7 +1773,7 @@ sub import_csv_file($) {
 
 				$log->debug("storing product", { code => $code, product_id => $product_id, org_id => $org_id, Owner_id => $Owner_id }) if $log->is_debug();
 
-				store_product($user_id, $product_ref, "Editing product (import) - " . $product_comment );
+				store_product($user_id, $product_ref, "Editing product (import) - " . ($product_comment || "") );
 
 				push @edited, $code;
 				$edited{$code}++;

lib/ProductOpener/ImportConvert.pm

@@ -114,6 +114,7 @@ use Time::Local;
 use Data::Dumper;
 use Text::CSV;
 use HTML::Entities qw(decode_entities);
+use XML::Rules;
 
 %fields = ();
 @fields = ();
@@ -291,14 +292,14 @@ sub assign_countries_for_product($$$) {
 	my $lcs_ref = shift;
 	my $default_country = shift;
 
-	foreach my $possible_lc (keys %{$lcs_ref}) {
+	foreach my $possible_lc (sort keys %{$lcs_ref}) {
 		if (defined $product_ref->{"product_name_" . $possible_lc}) {
 			assign_value($product_ref,"countries", $lcs_ref->{$possible_lc});
 			$log->info("assign_countries_for_product: found lc - assigning value", { lc => $possible_lc, countries => $lcs_ref->{$possible_lc}}) if $log->is_info();
 		}
 	}
 
-	if (not defined $product_ref->{countries}) {
+	if ((not defined $product_ref->{countries}) or ($product_ref->{countries} eq "")) {
 		assign_value($product_ref,"countries", $default_country);
 		$log->info("assign_countries_for_product: assigning default value", { countries => $default_country}) if $log->is_info();
 	}
@@ -536,14 +537,15 @@ sub remove_quantity_from_field($$) {
 		my $quantity = $product_ref->{quantity};
 		my $quantity_value = $product_ref->{quantity_value};
 		my $quantity_unit = $product_ref->{quantity_unit};
-
-		$quantity =~ s/\(/\\\(/g;
-		$quantity =~ s/\)/\\\)/g;
-		$quantity =~ s/\[/\\\[/g;
-		$quantity =~ s/\]/\\\]/g;
 
-		if ((defined $quantity) and ($product_ref->{$field} =~ /\s*(\b|\s+)($quantity|(\(|\[)$quantity(\)|\]))\s*$/i)) {
-			$product_ref->{$field} = $`;
+		if (defined $quantity) {
+			$quantity =~ s/\(/\\\(/g;
+			$quantity =~ s/\)/\\\)/g;
+			$quantity =~ s/\[/\\\[/g;
+			$quantity =~ s/\]/\\\]/g;
+			if ($product_ref->{$field} =~ /\s*(\b|\s+)($quantity|(\(|\[)$quantity(\)|\]))\s*$/i) {
+				$product_ref->{$field} = $`;
+			}
 		}
 		elsif ((defined $quantity_value) and (defined $quantity_unit) and ($product_ref->{$field} =~ /\s*\b\(?$quantity_value $quantity_unit\)?\s*$/i)) {
 			$product_ref->{$field} = $`;
@@ -870,6 +872,11 @@ sub clean_fields($) {
 
 		$log->debug("clean_fields", { field=>$field, value=>$product_ref->{$field} }) if $log->is_debug();
 
+		if (not defined $product_ref->{$field}) {
+			print STDERR "undefined value for field $field\n";
+			next;
+		}
+
 		# HTML entities
 		# e.g. Pâtes alimentaires cuites aromatisées au curcuma
 		if ($product_ref->{$field} =~ /\&/) {
@@ -1384,7 +1391,7 @@ sub load_xml_file($$$$) {
 			# multiple values in different languages
 
 			elsif ($source_tag eq '*') {
-				foreach my $tag ( keys %{$current_tag}) {
+				foreach my $tag ( sort keys %{$current_tag}) {
 					my $tag_target = $target;
 
 					# special case where we have something like allergens.nuts = traces
@@ -1776,12 +1783,14 @@ sub get_list_of_files(@) {
 
 
 
-sub print_csv_file() {
+sub print_csv_file($) {
+
+	my $file_handle = shift;
 
-	my $csv_out = Text::CSV->new ( { binary => 1 , sep_char => "\t" } )  # should set binary attribute.
+	my $csv_out = Text::CSV->new ( { binary => 1 , sep_char => "\t", eol => "\n", quote_space => 0 } )  # should set binary attribute.
                  or die "Cannot use CSV: ".Text::CSV->error_diag ();
 
-	print join("\t", @fields) . "\n";
+	$csv_out->print ($file_handle, \@fields) ;
 
 	foreach my $code (sort keys %products) {
 
@@ -1797,8 +1806,7 @@ sub print_csv_file() {
 			}
 		}
 
-		$csv_out->print (*STDOUT, \@values) ;
-		print "\n";
+		$csv_out->print ($file_handle, \@values) ;
 
 		print STDERR "code: $code\n";
 	}