diff --git a/cpanfile b/cpanfile index 941b8899bb005..501b7601ab453 100644 --- a/cpanfile +++ b/cpanfile @@ -60,6 +60,7 @@ requires 'File::chmod::Recursive'; # deps: libfile-chmod-perl requires 'Devel::Size'; # deps: libdevel-size-perl requires 'JSON::Create'; requires 'JSON::Parse'; +requires 'Data::DeepAccess'; # Mojolicious/Minion requires 'Mojolicious::Lite'; diff --git a/lib/ProductOpener/Config_off.pm b/lib/ProductOpener/Config_off.pm index 4305e768d3271..61c38cfab490d 100644 --- a/lib/ProductOpener/Config_off.pm +++ b/lib/ProductOpener/Config_off.pm @@ -902,36 +902,62 @@ $options{display_tag_additives} = [ ]; +# Used in the data_sources field (e.g. "App - Open Food Facts") +$options{apps_names} = { + + "elcoco" => "El CoCo", + "ethic-advisor" => "Ethic-Advisor", + "horizon" => "Horizon", + "infood" => "InFood", + "isve" => "IsVe", + "labeleat" => "LabelEat", + "off" => "Open Food Facts", + "scanfood" => "Scanfood", + "speisekammer" => "Speisekammer", + "waistline" => "Waistline", + "yuka" => "Yuka" +}; # Specific users used by apps $options{apps_userids} = { - "ethic-advisor" => "ethic-advisor", + "averment" => "isve", "elcoco" => "elcoco", + "ethic-advisor" => "ethic-advisor", + "inf" => "infood", "kiliweb" => "yuka", "labeleat" => "labeleat", + "prepperapp" => "speisekammer", + "scanfood" => "scanfood", + "swipe-studio" => "horizon", "waistline-app" => "waistline", - "inf" => "infood", }; +$options{official_app_id} = "off"; +$options{official_app_comment} = "(official (off|open food facts|openfoodfacts)|(off|open food facts|openfoodfacts) (official )?(android |ios )?(official )?app)"; + # (app)Official Android app 3.1.5 ( Added by 58abc55ceb98da6625cee5fb5feaf81 ) # (app)Labeleat1.0-SgP5kUuoerWvNH3KLZr75n6RFGA0 # (app)Contributed using: OFF app for iOS - v3.0 - user id: 3C0154A0-D19B-49EA-946F-CC33A05E404A -# (app)Official Android app 3.1.5 ( Added by 58abc55ceb98da6625cee5fb5feaf81 ) # (app)EthicAdvisorApp-production-2.6.3-user_17cf91e3-52ee-4431-aebf-7d455dd610f0 # (app)El Coco - user b0e8d6a858034cc750136b8f19a8953d +# app_uuid_prefix must be present to recognize the uuid, if the comment starts with the uuid, put an empty string $options{apps_uuid_prefix} = { "elcoco" => " user ", "ethic-advisor" => "user_", - "kiliweb" => "User :", "labeleat" => "Labeleat([^-]*)-", - "waistline-app" => "Waistline:", + "off" => "added by", + "scanfood" => "", + "waistline" => "Waistline:", + "yuka" => "User :", }; -$options{official_app_id} = "off"; -$options{official_app_comment} = "(official android app|off app)"; +$options{apps_uuid_suffix} = { + + "scanfood" => "scanfood", +}; $options{nova_groups_tags} = { diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index cba3030fb4aa0..4ed7a5ddc623c 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -183,6 +183,7 @@ use Excel::Writer::XLSX; use Template; use Data::Dumper; use Devel::Size qw(size total_size); +use Data::DeepAccess qw(deep_get); use Log::Log4perl; use Log::Any '$log', default_adapter => 'Stderr'; @@ -3880,6 +3881,19 @@ HTML display_error(lang("error_unknown_org"), 404); } } + elsif ($tagid =~ /\./) { + # App user (format "[app id].[app uuid]") + + my $appid = $`; + my $uuid = $'; + + my $app_name = deep_get(\%options, "apps_names", $appid) || $appid; + my $app_user = f_lang("f_app_user", { app_name => $app_name }); + + $title = $app_user; + $products_title = $app_user; + $display_tag = $app_user; + } else { # User diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm index 0870ba403a555..a74e867ee8e34 100644 --- a/lib/ProductOpener/Products.pm +++ b/lib/ProductOpener/Products.pm @@ -134,6 +134,7 @@ use ProductOpener::Text qw/:all/; use CGI qw/:cgi :form escapeHTML/; use Encode; use Log::Any qw($log); +use Data::DeepAccess qw(deep_get); use LWP::UserAgent; use Storable qw(dclone); @@ -1088,7 +1089,7 @@ sub store_product($$$) { delete $product_ref->{owners_tags}; } - push @{$changes_ref}, { + my $change_ref = { userid => $user_id, ip => remote_addr(), t => $product_ref->{last_modified_t}, @@ -1096,6 +1097,26 @@ sub store_product($$$) { rev => $rev, }; + # Allow apps to send the user agent as a form parameter instead of a HTTP header, as some web based apps can't change the User-Agent header sent by the browser + my $user_agent = remove_tags_and_quote(decode utf8=>param("User-Agent")) + || remove_tags_and_quote(decode utf8=>param("user-agent")) + || remove_tags_and_quote(decode utf8=>param("user_agent")) + || user_agent(); + + if ((defined $user_agent) and ($user_agent ne "")) { + $change_ref->{user_agent} = $user_agent; + } + + # Allow apps to send app_name, app_version and app_uuid parameters + foreach my $field (qw(app_name app_version app_uuid)) { + my $value = remove_tags_and_quote(decode utf8=>param($field)); + if ((defined $value) and ($value ne "")) { + $change_ref->{$field} = $value; + } + } + + push @{$changes_ref}, $change_ref; + add_user_teams($product_ref); compute_codes($product_ref); @@ -1106,7 +1127,7 @@ sub store_product($$$) { compute_product_history_and_completeness($new_data_root, $product_ref, $changes_ref, $blame_ref); - compute_data_sources($product_ref); + compute_data_sources($product_ref, $changes_ref); compute_main_countries($product_ref); @@ -1132,7 +1153,7 @@ sub store_product($$$) { # make sure nutrient values are numbers make_sure_numbers_are_stored_as_numbers($product_ref); - my $change_ref = $changes_ref->[-1]; + $change_ref = $changes_ref->[-1]; my $diffs = $change_ref->{diffs}; my %diffs = %{$diffs}; if ((!$diffs) or (!keys %diffs)) { @@ -1167,13 +1188,21 @@ sub store_product($$$) { return 1; } -# Update the data-sources tag from the sources field -# This function is for historic products, new sources should set the data_sources_tags field directly -# through import_csv_file.pl / upload_photos.pl etc. -sub compute_data_sources($) { +=head2 compute_data_sources ( $product_ref, $changes_ref ) + +Analyze the sources field of the product, as well as the changes to add to the data_sources field. + +Sources allows to add some producers imports that were done before the producers platform was created. + +The changes structure allows to add apps. + +=cut + +sub compute_data_sources($$) { my $product_ref = shift; + my $changes_ref = shift; my %data_sources = (); @@ -1241,16 +1270,14 @@ sub compute_data_sources($) { # Add a data source for apps - if (defined $product_ref->{editors_tags}) { - foreach my $editor (@{$product_ref->{editors_tags}}) { - - if ($editor =~ /\./) { + foreach my $change_ref (@$changes_ref) { + + if (defined $change_ref->{app}) { - my $app = $`; + my $app_name = deep_get(\%options, "apps_names", $change_ref->{app}) || $change_ref->{app}; - $data_sources{"Apps"} = 1; - $data_sources{"App - $app"} = 1; - } + $data_sources{"Apps"} = 1; + $data_sources{"App - " . $app_name} = 1; } } @@ -1476,22 +1503,45 @@ sub compute_completeness_and_missing_tags($$$) { } +=head2 get_change_userid_or_uuid ( $change_ref ) + +For a specific change, analyze change identifiers (comment, user agent, userid etc.) +to determine if the change was done through an app, the OFF userid, or an app specific UUID + +=cut + sub get_change_userid_or_uuid($) { my $change_ref = shift; my $userid = $change_ref->{userid}; - my $app = ""; + my $app; + my $app_userid_prefix; my $uuid; - if ((defined $userid) and (defined $options{apps_userids}) and (defined $options{apps_userids}{$userid})) { - $app = $options{apps_userids}{$userid} . "\."; + # Is it an app that sent a app_name? + if (defined $change_ref->{app_name}) { + $app = get_string_id_for_lang("no_language", $change_ref->{app_name}); } + # or is the userid specific to an app? + elsif (defined $userid) { + $app = deep_get(\%options, "apps_userids", $userid); + } + + # If the userid is an an account for an app, unset the userid, + # so that it can be replaced by the app + an app uuid if provided + if (defined $app) { + $userid = undef; + } + # Set the app field for the Open Food Facts app elsif ((defined $options{official_app_comment}) and ($change_ref->{comment} =~ /$options{official_app_comment}/i)) { - $app = $options{official_app_id} . "\."; + $app = $options{official_app_id}; } + # If we do not have a user specific userid (e.g. a logged in user using the Open Food Facts app), + # try to identify the UUID passed in the comment by some apps + # use UUID provided by some apps like Yuka # UUIDs are mix of [a-zA-Z0-9] chars, they must not be lowercased by getfile_id @@ -1501,21 +1551,54 @@ sub get_change_userid_or_uuid($) { # # but not: # (app)Updated via Power User Script - if ((defined $userid) and (defined $options{apps_uuid_prefix}) and (defined $options{apps_uuid_prefix}{$userid}) - and ($change_ref->{comment} =~ /$options{apps_uuid_prefix}{$userid}/i)) { - $uuid = $'; - } - if ((defined $uuid) and ($uuid !~ /^(\s|-|_|\.)*$/)) { - $uuid =~ s/^(\s*)//; - $uuid =~ s/(\s*)$//; - $userid = $app . $uuid; + if ((defined $app) and ((not defined $userid) or ($userid eq ''))) { + + $app_userid_prefix = deep_get(\%options, "apps_uuid_prefix", $app); + + # Check if the app passed the app_uuid parameter + if (defined $change_ref->{app_uuid}) { + $uuid = $change_ref->{app_uuid}; + } + # Extract UUID from comment + elsif ((defined $app_userid_prefix) + and ($change_ref->{comment} =~ /$app_userid_prefix/i)) { + $uuid = $'; + } + + if (defined $uuid) { + + # Remove any app specific suffix + my $app_userid_suffix = deep_get(\%options, "apps_uuid_suffix", $app); + if (defined $app_userid_suffix) { + $uuid =~ s/$app_userid_suffix(\s|\(|\[])*$//i; + } + + $uuid =~ s/^(-|_|\s|\(|\[])+//; + $uuid =~ s/(-|_|\s|\)|\])+$//; + } + + # If we have a uuid from an app, make the userid a combination of app + uuid + if ((defined $uuid) and ($uuid !~ /^(-|_|\s|-|_|\.)*$/)) { + $userid = $app . '.' . $uuid; + } + # otherwise use the original userid used for the API if any + elsif (defined $change_ref->{userid}) { + $userid = $change_ref->{userid}; + } } - if ((not defined $userid) or ($userid eq '')) { + if (not defined $userid) { $userid = "openfoodfacts-contributors"; + } + + # Add the app to the change structure if we identified one, this will be used to populate the data sources field + if (defined $app) { + $change_ref->{app} = $app; } + $log->debug("get_change_userid_or_uuid", { change_ref => $change_ref, app => $app, app_userid_prefix => $app_userid_prefix, uuid => $uuid, userid => $userid } ) if $log->is_debug(); + return $userid; } diff --git a/lib/ProductOpener/Tags.pm b/lib/ProductOpener/Tags.pm index 44b29ecfaeabf..d768ac2af1569 100644 --- a/lib/ProductOpener/Tags.pm +++ b/lib/ProductOpener/Tags.pm @@ -1756,8 +1756,11 @@ sub generate_tags_taxonomy_extract ($$$$) { my $options_ref = shift; my $lcs_ref = shift; + $log->debug("generate_tags_taxonomy_extract", {tagtype => $tagtype, tags_ref => $tags_ref, options_ref => $options_ref, lcs_ref => $lcs_ref }) if $log->is_debug(); + # Return empty hash if the taxonomy does not exist if (not defined $translations_to{$tagtype}) { + $log->debug("taxonomy not found", {tagtype => $tagtype}) if $log->is_debug(); return {}; } diff --git a/po/common/common.pot b/po/common/common.pot index 94729b2e8a560..a4b8d824e1bd9 100644 --- a/po/common/common.pot +++ b/po/common/common.pot @@ -5960,3 +5960,8 @@ msgstr "This might not be the answer people want to hear, but there is no safe l msgctxt "source" msgid "Source" msgstr "Source" + +# variable names between { } must not be translated +msgctxt "f_app_user" +msgid "A user of the {app_name} app" +msgstr "A user of the {app_name} app" \ No newline at end of file diff --git a/po/common/en.po b/po/common/en.po index 819f5b303a0f9..1ab39fd089ab3 100644 --- a/po/common/en.po +++ b/po/common/en.po @@ -5988,3 +5988,8 @@ msgstr "This might not be the answer people want to hear, but there is no safe l msgctxt "source" msgid "Source" msgstr "Source" + +# variable names between { } must not be translated +msgctxt "f_app_user" +msgid "A user of the {app_name} app" +msgstr "A user of the {app_name} app" \ No newline at end of file diff --git a/scripts/update_all_products.pl b/scripts/update_all_products.pl index e0654973769db..a5b0aeb8c2777 100755 --- a/scripts/update_all_products.pl +++ b/scripts/update_all_products.pl @@ -526,7 +526,7 @@ $product_ref->{rev} = $last_rev; my $blame_ref = {}; compute_product_history_and_completeness($data_root, $product_ref, $changes_ref, $blame_ref); - compute_data_sources($product_ref); + compute_data_sources($product_ref, $changes_ref); store("$data_root/products/$path/changes.sto", $changes_ref); } else { @@ -831,7 +831,11 @@ } if ($compute_data_sources) { - compute_data_sources($product_ref); + my $changes_ref = retrieve("$data_root/products/$path/changes.sto"); + if (not defined $changes_ref) { + $changes_ref = []; + } + compute_data_sources($product_ref, $changes_ref); } if ($compute_nova) { @@ -1016,7 +1020,7 @@ } my $blame_ref = {}; compute_product_history_and_completeness($data_root, $product_ref, $changes_ref, $blame_ref); - compute_data_sources($product_ref); + compute_data_sources($product_ref, $changes_ref); store("$data_root/products/$path/changes.sto", $changes_ref); } diff --git a/t/products.t b/t/products.t index bf78bf7b6cda4..2b2885fa49fc0 100644 --- a/t/products.t +++ b/t/products.t @@ -92,19 +92,81 @@ foreach my $field (@string_fields) { compute_and_test_completeness($product_ref, 1.0, 'product all fields'); +# Test the function that recognizes the app and app uuid from changes and sets the app and userid + my @get_change_userid_or_uuid_tests = ( -["real-user", "some random comment", "real-user"], -["stephane", "Updated via Power User Script", "stephane"], -["kiliweb", "User : WjR3OExvb3M5dWNobU1Za29EUHJvdmtwN0p1SVh6MjNDZGdySVE9PQ", "yuka.WjR3OExvb3M5dWNobU1Za29EUHJvdmtwN0p1SVh6MjNDZGdySVE9PQ"], -); -foreach my $test_ref (@get_change_userid_or_uuid_tests) { + { + userid => undef, + comment => "some random comment", + expected_app => undef, + expected_userid => "openfoodfacts-contributors", + }, + { + userid => "real-user", + comment => "some random comment", + expected_app => undef, + expected_userid => "real-user", + }, + { + userid => "stephane", + comment => "Updated via Power User Script", + expected_app => undef, + expected_userid => "stephane", + }, + { + userid => "stephane", + comment => "Official Open Food Facts Android app 3.6.6 (Added by a99a030f-c836-4551-9ec7-3d387f293e73)", + expected_app => "off", + expected_userid => "stephane", + }, + { + userid => undef, + comment => "Official Open Food Facts Android app 3.6.6 (Added by a99a030f-c836-4551-9ec7-3d387f293e73)", + expected_app => "off", + expected_userid => "off.a99a030f-c836-4551-9ec7-3d387f293e73", + }, + { + userid => "kiliweb", + comment => "User : WjR3OExvb3M5dWNobU1Za29EUHJvdmtwN0p1SVh6MjNDZGdySVE9PQ", + expected_app => "yuka", + expected_userid => "yuka.WjR3OExvb3M5dWNobU1Za29EUHJvdmtwN0p1SVh6MjNDZGdySVE9PQ", + }, + { + userid => "prepperapp", + comment => "Edited by a user of https://speisekammer-app.de", + expected_app => "speisekammer", + expected_userid => "prepperapp", + }, + { + userid => "scanfood", + comment => "96ce87ae-2f2b-4fd6-90d2-7bfc4388d173-ScanFood", + expected_app => "scanfood", + expected_userid => "scanfood.96ce87ae-2f2b-4fd6-90d2-7bfc4388d173", + }, + { + userid => "someuser", + comment => "some comment", + app_name => "Some App", + expected_app => "some-app", + expected_userid => "someuser", + }, + { + userid => "someuser", + comment => "some comment", + app_name => "Some App", + app_uuid => "423T42fFST423", + expected_app => "some-app", + expected_userid => "some-app.423T42fFST423", + }, +); - my $change_ref = { userid => $test_ref->[0], comment => $test_ref->[1] }; +foreach my $change_ref (@get_change_userid_or_uuid_tests) { - my $userid = get_change_userid_or_uuid($change_ref); + $change_ref->{resulting_userid} = get_change_userid_or_uuid($change_ref); - is ($userid, $test_ref->[2]) or diag explain $test_ref; + is ($change_ref->{app}, $change_ref->{expected_app}) or diag explain $change_ref; + is ($change_ref->{resulting_userid}, $change_ref->{expected_userid}) or diag explain $change_ref; }