feat: migration script to batch import existing orgs in CRM (#10683)

<!-- IMPORTANT CHECKLIST
Make sure you've done all the following (You can delete the checklist
before submitting)
- [ ] PR title is prefixed by one of the following: feat, fix, docs,
style, refactor, test, build, ci, chore, revert, l10n, taxonomy
- [ ] Code is well documented
- [ ] Include unit tests for new functionality
- [ ] Code passes GitHub workflow checks in your branch
- [ ] If you have multiple commits please combine them into one commit
by squashing them.
- [ ] Read and understood the [contribution
guidelines](https://github.com/openfoodfacts/openfoodfacts-server/blob/main/CONTRIBUTING.md)
-->
### What

Existing orgs have been manually reviewed by @manon-corneille, using a
dump generated by this script
#10507.
This PR provides a migration script
(`scripts/migration/2024_08_validate_and_sync_crm_existing_orgs.pl`) to
accept the selected orgs. The others are rejected and receive a
rejection email notification.

- added sync of org's country
- few fixes in CRM.pm 

- [x]  Awaiting email's wording.


### Screenshot
<!-- Optional, you can delete if not relevant -->

### Related issue(s) and discussion

- Fixes #7355
- Fixes #7359

lib/ProductOpener/CRM.pm

@@ -82,7 +82,7 @@ BEGIN {
 use vars @EXPORT_OK;
 
 use ProductOpener::Config2;
-use ProductOpener::Tags qw/%country_codes_reverse/;
+use ProductOpener::Tags qw/country_to_cc/;
 use ProductOpener::Users qw/retrieve_user store_user/;
 use ProductOpener::Orgs qw/retrieve_org is_user_in_org_group/;
 use ProductOpener::Paths qw/%BASE_DIRS ensure_dir_created/;
@@ -126,6 +126,8 @@ sub find_or_create_contact($user_ref) {
 	my $contact_id = find_contact($user_ref);
 	if (defined $contact_id) {
 		return if not link_user_with_contact($user_ref, $contact_id);
+		add_category_to_partner($user_ref, 'Producer');
+		update_partner_country($user_ref);
 	}
 	else {
 		$contact_id = create_contact($user_ref);
@@ -223,17 +225,9 @@ sub create_contact ($user_ref) {
 		email => $user_ref->{email},
 		phone => $user_ref->{phone},
 		category_id => [$crm_data->{category}{Producer}],
+		country_id => _get_country_id($user_ref->{country}),
 	};
 
-	# find country code id in Odoo
-	my $user_country_code = uc($country_codes_reverse{$user_ref->{country}}) || 'EN';
-	my $country_id = make_odoo_request(
-		'res.country', 'search_read',
-		[[['code', '=', $user_country_code]]],
-		{fields => ['code', 'id']}
-	);
-	$contact->{country_id} = $country_id->[0]{id};
-
 	# find spoken language's code in Odoo
 	# 'lang' are actived languages in Odoo
 	my $Odoo_lang = make_odoo_request(
@@ -253,6 +247,22 @@ sub create_contact ($user_ref) {
 	return $contact_id;
 }
 
+=head1 _get_country ($tag)
+
+Get the country from a tag (en:france, en:germany, ...)
+
+=cut
+
+sub _get_country_id($country_tag) {
+	my $country_code = uc(country_to_cc($country_tag));
+	my $country_id
+		= make_odoo_request('res.country', 'search_read', [[['code', '=', $country_code]]], {fields => ['code', 'id']});
+	if (scalar @$country_id) {
+		return $country_id->[0]{id};
+	}
+	return;
+}
+
 =head2 find_or_create_company ($org_ref, $contact_id = undef)
 
 Attempts to find a company, and if it doesn't exist, creates it
@@ -277,6 +287,8 @@ sub find_or_create_company($org_ref, $contact_id = undef) {
 	my $company_id = find_company($org_ref, $contact_id);
 	if (defined $company_id) {
 		return if not link_org_with_company($org_ref, $company_id);
+		add_category_to_partner($org_ref, 'Producer');
+		update_partner_country($org_ref);
 	}
 	else {
 		$company_id = create_company($org_ref);
@@ -411,6 +423,7 @@ sub create_company ($org_ref) {
 		is_company => 1,
 		x_off_org_id => $org_ref->{org_id},
 		x_off_main_contact => $main_contact_user_ref->{crm_user_id},
+		country_id => _get_country_id($org_ref->{country})
 	};
 	my $company_id = make_odoo_request('res.partner', 'create', [{%$company}]);
 	$log->debug("create_company", {company_id => $company_id, company => $company}) if $log->is_debug();
@@ -549,7 +562,7 @@ id of a member of the organization
 
 =cut
 
-sub change_company_main_contact($org_ref, $user_id) {
+sub change_company_main_contact ($org_ref, $user_id) {
 
 	if (not is_user_in_org_group($org_ref, $user_id, 'members')) {
 		$log->error("change_company_main_contact",
@@ -581,19 +594,19 @@ sub change_company_main_contact($org_ref, $user_id) {
 	return $req_company;
 }
 
-sub update_last_import_date($org_ref, $time) {
+sub update_last_import_date ($org_ref, $time) {
 	return _update_partner_field($org_ref, 'x_off_last_import_date', _time_to_odoo_date_str($time));
 }
 
-sub update_last_export_date($org_ref, $time) {
+sub update_last_export_date ($org_ref, $time) {
 	return _update_partner_field($org_ref, 'x_off_last_export_date', _time_to_odoo_date_str($time));
 }
 
-sub update_public_products($org_ref, $number_of_products) {
+sub update_public_products ($org_ref, $number_of_products) {
 	return _update_partner_field($org_ref, 'x_off_public_products', $number_of_products);
 }
 
-sub update_pro_products($org_ref, $number_of_products) {
+sub update_pro_products ($org_ref, $number_of_products) {
 	return _update_partner_field($org_ref, 'x_off_pro_products', $number_of_products);
 }
 
@@ -607,11 +620,16 @@ sub update_template_download_date ($org_id) {
 	return _update_partner_field($org_ref, 'x_off_last_template_download_date', _time_to_odoo_date_str(time()));
 }
 
-sub update_company_last_logged_in_contact($org_ref, $user_ref) {
+sub update_company_last_logged_in_contact ($org_ref, $user_ref) {
 	return _update_partner_field($org_ref, 'x_off_last_logged_org_contact', $user_ref->{crm_user_id});
 }
 
-sub _update_partner_field($user_or_org, $field, $value) {
+sub update_partner_country ($user_or_org_ref) {
+	my $country_id = _get_country_id($user_or_org_ref->{country});
+	return _update_partner_field($user_or_org_ref, 'country_id', $country_id);
+}
+
+sub _update_partner_field ($user_or_org, $field, $value) {
 	my $partner_id = $user_or_org->{crm_user_id} // $user_or_org->{crm_org_id};
 	return if not defined $partner_id;
 
@@ -621,31 +639,34 @@ sub _update_partner_field($user_or_org, $field, $value) {
 	return make_odoo_request('res.partner', 'write', [[$partner_id], {$field => $value}]);
 }
 
-sub _time_to_odoo_date_str($time) {
+sub _time_to_odoo_date_str ($time) {
 	my ($sec, $min, $hour, $mday, $mon, $year) = localtime($time);
 	$year += 1900;
 	$mon += 1;
 	return sprintf("%04d-%02d-%02d", $year, $mon, $mday);
 }
 
-=head2 add_category_to_company ($org_id, $label)
+=head2 add_category_to_partner ($contact_or_org_ref, $label)
 
-Add a category to a company in Odoo
+Add a category to a partner (contact or company) in Odoo
 
 =head3 Arguments
 
 =head4 $org_id
 
 =head4 $label
 
+C<$label> must match one of the values in C<@required_category_labels>
+
 =head3 Return values
 
 1 if success, undef otherwise
 
 =cut
 
-sub add_category_to_company($org_ref, $label) {
-	return if not defined $org_ref->{crm_org_id};
+sub add_category_to_partner($contact_or_org_ref, $label) {
+	my $partner_id = $contact_or_org_ref->{crm_user_id} // $contact_or_org_ref->{crm_org_id};
+	return if not defined $partner_id;
 
 	my $category_id = $crm_data->{category}{$label};
 	return if not defined $category_id;
@@ -654,10 +675,10 @@ sub add_category_to_company($org_ref, $label) {
 			if $log->is_debug();
 		return;
 	}
-	$log->debug("add_category_to_company", {org_id => $org_ref->{org_id}, label => $label, category_id => $category_id})
+	$log->debug("add_category_to_company", {partner_id => $partner_id, label => $label, category_id => $category_id})
 		if $log->is_debug();
 	return make_odoo_request('res.partner', 'write',
-		[[$org_ref->{crm_org_id}], {category_id => [[$commands{link}, $category_id]]}]);
+		[[$partner_id], {category_id => [[$commands{link}, $category_id]]}]);
 }
 
 =head2 update_company_last_import_type ($org_id, $data_source)
@@ -673,7 +694,7 @@ must match one of the values in CRM.pm @data_source
 sub update_company_last_import_type($org_ref, $label) {
 	return if not defined $org_ref->{crm_org_id};
 	my $category_id = $crm_data->{category}{$label};
-	add_category_to_company($org_ref, $label);
+	add_category_to_partner($org_ref, $label);
 	return make_odoo_request('res.partner', 'write',
 		[[$org_ref->{crm_org_id}], {x_off_last_import_type => $category_id}]);
 }

lib/ProductOpener/Orgs.pm

@@ -61,6 +61,7 @@ BEGIN {
 		&update_last_logged_in_member
 		&update_last_import_type
 		&accept_pending_user_in_org
+		&send_rejection_email
 
 	);    # symbols to export on request
 	%EXPORT_TAGS = (all => [@EXPORT_OK]);
@@ -276,6 +277,7 @@ sub create_org ($creator, $org_id_or_name) {
 		admins => {},
 		members => {},
 		main_contact => undef,
+		country => $country,
 	};
 
 	store_org($org_ref);
@@ -502,6 +504,40 @@ sub update_export_date($org_id_or_ref, $time) {
 	return;
 }
 
+sub send_rejection_email ($org_ref) {
+	# send org rejection email to main contact
+	my $main_contact_user = $org_ref->{main_contact};
+	my $user_ref = retrieve_user($main_contact_user);
+	if (not defined $user_ref) {
+		$log->warning("send_rejection_email", {error => "main contact user not found", org_ref => $org_ref})
+			if $log->is_warning();
+		return;
+	}
+
+	my $language = $user_ref->{preferred_language} || $user_ref->{initial_lc};
+	# if template does not exist in the requested language, use English
+	my $template_name = "org_rejected.tt.html";
+	my $template_path = "emails/$language/$template_name";
+	my $default_path = "emails/en/$template_name";
+	my $path = -e "$data_root/templates/$template_path" ? $template_path : $default_path;
+
+	my $template_data_ref = {
+		user => $user_ref,
+		org => $org_ref,
+	};
+
+	my $email = '';
+	my $res = process_template($path, $template_data_ref, \$email);
+	if ($email =~ /^\s*Subject:\s*(.*)\n/i) {
+		my $subject = $1;
+		my $body = $';
+		$body =~ s/^\n+//;
+		send_html_email($user_ref, $subject, $email);
+	}
+	$log->debug("send_rejection_email", {path => $path, email => $email, res => $res}) if $log->is_debug();
+	return;
+}
+
 sub update_last_logged_in_member($user_ref) {
 
 	my $org_id = $user_ref->{org_id} // $user_ref->{requested_org_id};

scripts/migrations/2024_08_validate_and_sync_crm_existing_orgs.pl

@@ -0,0 +1,94 @@
+#!/usr/bin/perl -w
+
+# This file is part of Product Opener.
+#
+# Product Opener
+# Copyright (C) 2011-2023 Association Open Food Facts
+# Contact: contact@openfoodfacts.org
+# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France
+#
+# Product Opener is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use ProductOpener::PerlStandards;
+use Modern::Perl '2017';
+use utf8;
+
+use ProductOpener::Config qw( $data_root );
+use ProductOpener::Paths qw( %BASE_DIRS );
+use ProductOpener::Users qw( $User_id retrieve_user );
+use ProductOpener::Orgs qw( list_org_ids retrieve_org store_org send_rejection_email);
+use Encode;
+
+binmode(STDOUT, ":encoding(UTF-8)");
+binmode(STDERR, ":encoding(UTF-8)");
+
+# This file is used to:
+# Set the validation status of existing orgs to 'accepted' for the ones in the list, and sync with the CRM.
+# Reject all the others and send a rejection email notification to them.
+# Set the org name if it is missing.
+
+# Set Manon as the salesperson for the orgs
+$User_id = 'manoncorneille';
+
+# read the list of orgs to sync, one per line
+my %orgs_to_accept = map {chomp; $_ => 1} <>;
+
+# load checkpoint
+my $checkpoint_file = "$BASE_DIRS{CACHE_TMP}/orgs_synced.checkpoint";
+my $checkpoint;
+if (!-e $checkpoint_file) {
+	`touch $checkpoint_file`;
+}
+open($checkpoint, '+<:encoding(UTF-8)', $checkpoint_file) or die "Could not open file: $!";
+foreach my $org_id (<$checkpoint>) {
+	chomp $org_id;
+	delete $orgs_to_accept{$org_id};
+}
+
+# if all orgs have been synced, exit
+if (scalar keys %orgs_to_accept == 0) {
+	print "All orgs have been synced\n";
+	close $checkpoint;
+	exit;
+}
+
+foreach my $org_id (list_org_ids()) {
+	$org_id = decode utf8 => $org_id;
+	my $org_ref = retrieve_org($org_id);
+
+	my $org_name = $org_ref->{name};
+	if (not defined $org_name) {
+		$org_ref->{name} = $org_id =~ s/-/ /gr;
+	}
+
+	if (not exists $org_ref->{country} and exists $org_ref->{main_contact}) {
+		my $user_ref = retrieve_user($org_ref->{main_contact});
+		$org_ref->{country} = $user_ref->{country} if $user_ref;
+	}
+
+	my $org_is_valid = exists $orgs_to_accept{$org_id};
+	if ($org_is_valid) {
+		$org_ref->{valid_org} = 'accepted';
+	}
+	else {
+		$org_ref->{valid_org} = 'rejected';
+		send_rejection_email($org_ref);
+	}
+
+	store_org($org_ref);
+	if ($org_is_valid) {
+		print "$org_id\n";
+		print $checkpoint "$org_id\n";
+	}
+}

scripts/save_org_product_data_daily_off_pro.pl

@@ -151,7 +151,7 @@ ($org_id)
 
 	# sync crm
 	update_public_products($org_ref, $org_ref->{data}{products}{number_of_products_on_public_platform});
-	update_pro_products($org_ref), $org_ref->{data}{products}{number_of_products_on_producer_platform};
+	update_pro_products($org_ref, $org_ref->{data}{products}{number_of_products_on_producer_platform});
 
 	store($org_ref, $org_file_path);
 	return;

templates/emails/en/org_rejected.tt.html

@@ -0,0 +1,13 @@
+Subject: Account deactivation notification
+
+<!-- start templates/[% template.name %] -->
+
+Hello,<br><br>
+As part of our ongoing efforts to improve and maintain the quality of our platform, we recently conducted a review of all Open Food Facts Pro accounts. During this review, we identified certain accounts that either had not yet imported any product data or appeared to be registered with personal email addresses rather than professional ones.<br><br>
+Based on these criteria, <b>we have made the decision to deactivate your account</b>. However, if you believe this action was taken in error or if you are still interested in using the platform, we are more than happy to assist you in getting started with the Open Food Facts Pro platform.<br><br>
+If you would like to reactivate your account, please respond to this email, and we will guide you through the next steps. We want to ensure that everyone who wishes to contribute to our collaborative database has the opportunity to do so.<br><br>
+Thank you for your understanding, and we apologize for any inconvenience this may have caused.<br><br>
+Best,<br><br>
+The Open Food Facts team<br><br>
+
+<!-- end templates/[% template.name %] -->

templates/emails/fr/org_rejected.tt.html

@@ -0,0 +1,12 @@
+Subject: Désactivation de votre compte
+
+<!-- start templates/[% template.name %] -->
+
+Bonjour,<br><br>
+Dans le cadre de nos efforts continus pour améliorer la plateforme Pro, nous avons récemment passé en revue tous les comptes Open Food Facts Pro. Plusieurs comptes n'ont encore jamais importé de données produits ou semblent être enregistrés avec des adresses mail personnelles plutôt que professionnelles.<br><br>
+Sur la base de ces critères, nous avons pris la décision de désactiver votre compte. Toutefois, si vous pensez que cette mesure a été prise par erreur ou si vous êtes toujours intéressés par l'utilisation de la plateforme, nous serions ravis de vous aider à ajouter vos produits sur la plateforme Pro d'Open Food Facts.<br><br>
+Si vous souhaitez réactiver votre compte, merci de répondre à cet e-mail.<br><br>
+Nous vous remercions de votre compréhension,<br><br>
+L’équipe d’Open Food Facts<br><br>
+
+<!-- end templates/[% template.name %] -->

tests/integration/expected_test_results/create_pro_user/org-after-subscription.json

@@ -2,6 +2,7 @@
    "admins" : {
       "tests" : 1
    },
+   "country" : "en:world",
    "created_t" : "--ignore--",
    "creator" : "tests",
    "last_logged_member" : "tests",

tests/integration/expected_test_results/create_pro_user/org-after-validation.json

@@ -3,6 +3,7 @@
    "admins" : {
       "tests" : 1
    },
+   "country" : "en:world",
    "created_t" : "--ignore--",
    "creator" : "tests",
    "do_not_import_codeonline" : "",