Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Fix and improve detection of apps (name and UUID) to populate data sources #6319

Merged
merged 6 commits into from
Jan 21, 2022
Merged

feat: Fix and improve detection of apps (name and UUID) to populate data sources #6319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8e703d7
improve app and uuid detection in changes
stephanegigandet Jan 20, 2022
fefd24e
better tests
stephanegigandet Jan 20, 2022
00fa1b1
better identification of apps
stephanegigandet Jan 20, 2022
8e63282
add support for passing app_name and app_uuid through the API
stephanegigandet Jan 20, 2022
a76ea41
remove unused module
stephanegigandet Jan 21, 2022
db1bfbf
add Data::DeepAccess
stephanegigandet Jan 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cpanfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ requires 'File::chmod::Recursive'; # deps: libfile-chmod-perl
requires 'Devel::Size'; # deps: libdevel-size-perl
requires 'JSON::Create';
requires 'JSON::Parse';
requires 'Data::DeepAccess';

# Mojolicious/Minion
requires 'Mojolicious::Lite';
Expand Down
40 changes: 33 additions & 7 deletions lib/ProductOpener/Config_off.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -902,36 +902,62 @@ $options{display_tag_additives} = [

];

# Used in the data_sources field (e.g. "App - Open Food Facts")
$options{apps_names} = {

"elcoco" => "El CoCo",
"ethic-advisor" => "Ethic-Advisor",
"horizon" => "Horizon",
"infood" => "InFood",
"isve" => "IsVe",
"labeleat" => "LabelEat",
"off" => "Open Food Facts",
"scanfood" => "Scanfood",
"speisekammer" => "Speisekammer",
"waistline" => "Waistline",
"yuka" => "Yuka"
};

# Specific users used by apps
$options{apps_userids} = {

"ethic-advisor" => "ethic-advisor",
"averment" => "isve",
"elcoco" => "elcoco",
"ethic-advisor" => "ethic-advisor",
"inf" => "infood",
"kiliweb" => "yuka",
"labeleat" => "labeleat",
"prepperapp" => "speisekammer",
"scanfood" => "scanfood",
"swipe-studio" => "horizon",
"waistline-app" => "waistline",
"inf" => "infood",
};

$options{official_app_id} = "off";
$options{official_app_comment} = "(official (off|open food facts|openfoodfacts)|(off|open food facts|openfoodfacts) (official )?(android |ios )?(official )?app)";

# (app)Official Android app 3.1.5 ( Added by 58abc55ceb98da6625cee5fb5feaf81 )
# (app)Labeleat1.0-SgP5kUuoerWvNH3KLZr75n6RFGA0
# (app)Contributed using: OFF app for iOS - v3.0 - user id: 3C0154A0-D19B-49EA-946F-CC33A05E404A
# (app)Official Android app 3.1.5 ( Added by 58abc55ceb98da6625cee5fb5feaf81 )
# (app)EthicAdvisorApp-production-2.6.3-user_17cf91e3-52ee-4431-aebf-7d455dd610f0
# (app)El Coco - user b0e8d6a858034cc750136b8f19a8953d

# app_uuid_prefix must be present to recognize the uuid, if the comment starts with the uuid, put an empty string
$options{apps_uuid_prefix} = {

"elcoco" => " user ",
"ethic-advisor" => "user_",
"kiliweb" => "User :",
"labeleat" => "Labeleat([^-]*)-",
"waistline-app" => "Waistline:",
"off" => "added by",
"scanfood" => "",
"waistline" => "Waistline:",
"yuka" => "User :",
};

$options{official_app_id} = "off";
$options{official_app_comment} = "(official android app|off app)";
$options{apps_uuid_suffix} = {

"scanfood" => "scanfood",
};


$options{nova_groups_tags} = {
Expand Down
14 changes: 14 additions & 0 deletions lib/ProductOpener/Display.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ use Excel::Writer::XLSX;
use Template;
use Data::Dumper;
use Devel::Size qw(size total_size);
use Data::DeepAccess qw(deep_get);
use Log::Log4perl;

use Log::Any '$log', default_adapter => 'Stderr';
Expand Down Expand Up @@ -3880,6 +3881,19 @@ HTML
display_error(lang("error_unknown_org"), 404);
}
}
elsif ($tagid =~ /\./) {
# App user (format "[app id].[app uuid]")

my $appid = $`;
my $uuid = $';

my $app_name = deep_get(\%options, "apps_names", $appid) || $appid;
my $app_user = f_lang("f_app_user", { app_name => $app_name });

$title = $app_user;
$products_title = $app_user;
$display_tag = $app_user;
}
else {

# User
Expand Down
139 changes: 111 additions & 28 deletions lib/ProductOpener/Products.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ use ProductOpener::Text qw/:all/;
use CGI qw/:cgi :form escapeHTML/;
use Encode;
use Log::Any qw($log);
use Data::DeepAccess qw(deep_get);

use LWP::UserAgent;
use Storable qw(dclone);
Expand Down Expand Up @@ -1088,14 +1089,34 @@ sub store_product($$$) {
delete $product_ref->{owners_tags};
}

push @{$changes_ref}, {
my $change_ref = {
userid => $user_id,
ip => remote_addr(),
t => $product_ref->{last_modified_t},
comment => $comment,
rev => $rev,
};

# Allow apps to send the user agent as a form parameter instead of a HTTP header, as some web based apps can't change the User-Agent header sent by the browser
my $user_agent = remove_tags_and_quote(decode utf8=>param("User-Agent"))
|| remove_tags_and_quote(decode utf8=>param("user-agent"))
|| remove_tags_and_quote(decode utf8=>param("user_agent"))
|| user_agent();

if ((defined $user_agent) and ($user_agent ne "")) {
$change_ref->{user_agent} = $user_agent;
}

# Allow apps to send app_name, app_version and app_uuid parameters
foreach my $field (qw(app_name app_version app_uuid)) {
my $value = remove_tags_and_quote(decode utf8=>param($field));
if ((defined $value) and ($value ne "")) {
$change_ref->{$field} = $value;
}
}
Comment on lines +1110 to +1116
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So we have three string parameters:

  • app_name
  • app_version
  • app_uuid


push @{$changes_ref}, $change_ref;

add_user_teams($product_ref);

compute_codes($product_ref);
Expand All @@ -1106,7 +1127,7 @@ sub store_product($$$) {

compute_product_history_and_completeness($new_data_root, $product_ref, $changes_ref, $blame_ref);

compute_data_sources($product_ref);
compute_data_sources($product_ref, $changes_ref);

compute_main_countries($product_ref);

Expand All @@ -1132,7 +1153,7 @@ sub store_product($$$) {
# make sure nutrient values are numbers
make_sure_numbers_are_stored_as_numbers($product_ref);

my $change_ref = $changes_ref->[-1];
$change_ref = $changes_ref->[-1];
my $diffs = $change_ref->{diffs};
my %diffs = %{$diffs};
if ((!$diffs) or (!keys %diffs)) {
Expand Down Expand Up @@ -1167,13 +1188,21 @@ sub store_product($$$) {
return 1;
}

# Update the data-sources tag from the sources field
# This function is for historic products, new sources should set the data_sources_tags field directly
# through import_csv_file.pl / upload_photos.pl etc.

sub compute_data_sources($) {
=head2 compute_data_sources ( $product_ref, $changes_ref )

Analyze the sources field of the product, as well as the changes to add to the data_sources field.

Sources allows to add some producers imports that were done before the producers platform was created.

The changes structure allows to add apps.

=cut

sub compute_data_sources($$) {

my $product_ref = shift;
my $changes_ref = shift;

my %data_sources = ();

Expand Down Expand Up @@ -1241,16 +1270,14 @@ sub compute_data_sources($) {

# Add a data source for apps

if (defined $product_ref->{editors_tags}) {
foreach my $editor (@{$product_ref->{editors_tags}}) {

if ($editor =~ /\./) {
foreach my $change_ref (@$changes_ref) {

if (defined $change_ref->{app}) {

my $app = $`;
my $app_name = deep_get(\%options, "apps_names", $change_ref->{app}) || $change_ref->{app};

$data_sources{"Apps"} = 1;
$data_sources{"App - $app"} = 1;
}
$data_sources{"Apps"} = 1;
$data_sources{"App - " . $app_name} = 1;
}
}

Expand Down Expand Up @@ -1476,22 +1503,45 @@ sub compute_completeness_and_missing_tags($$$) {
}


=head2 get_change_userid_or_uuid ( $change_ref )

For a specific change, analyze change identifiers (comment, user agent, userid etc.)
to determine if the change was done through an app, the OFF userid, or an app specific UUID

=cut

sub get_change_userid_or_uuid($) {

my $change_ref = shift;

my $userid = $change_ref->{userid};

my $app = "";
my $app;
my $app_userid_prefix;
my $uuid;

if ((defined $userid) and (defined $options{apps_userids}) and (defined $options{apps_userids}{$userid})) {
$app = $options{apps_userids}{$userid} . "\.";
# Is it an app that sent a app_name?
if (defined $change_ref->{app_name}) {
$app = get_string_id_for_lang("no_language", $change_ref->{app_name});
}
# or is the userid specific to an app?
elsif (defined $userid) {
$app = deep_get(\%options, "apps_userids", $userid);
}

# If the userid is an an account for an app, unset the userid,
# so that it can be replaced by the app + an app uuid if provided
if (defined $app) {
$userid = undef;
}
# Set the app field for the Open Food Facts app
elsif ((defined $options{official_app_comment}) and ($change_ref->{comment} =~ /$options{official_app_comment}/i)) {
$app = $options{official_app_id} . "\.";
$app = $options{official_app_id};
}

# If we do not have a user specific userid (e.g. a logged in user using the Open Food Facts app),
# try to identify the UUID passed in the comment by some apps

# use UUID provided by some apps like Yuka
# UUIDs are mix of [a-zA-Z0-9] chars, they must not be lowercased by getfile_id

Expand All @@ -1501,21 +1551,54 @@ sub get_change_userid_or_uuid($) {
#
# but not:
# (app)Updated via Power User Script
if ((defined $userid) and (defined $options{apps_uuid_prefix}) and (defined $options{apps_uuid_prefix}{$userid})
and ($change_ref->{comment} =~ /$options{apps_uuid_prefix}{$userid}/i)) {
$uuid = $';
}

if ((defined $uuid) and ($uuid !~ /^(\s|-|_|\.)*$/)) {
$uuid =~ s/^(\s*)//;
$uuid =~ s/(\s*)$//;
$userid = $app . $uuid;
if ((defined $app) and ((not defined $userid) or ($userid eq ''))) {

$app_userid_prefix = deep_get(\%options, "apps_uuid_prefix", $app);

# Check if the app passed the app_uuid parameter
if (defined $change_ref->{app_uuid}) {
$uuid = $change_ref->{app_uuid};
}
# Extract UUID from comment
elsif ((defined $app_userid_prefix)
and ($change_ref->{comment} =~ /$app_userid_prefix/i)) {
$uuid = $';
}

if (defined $uuid) {

# Remove any app specific suffix
my $app_userid_suffix = deep_get(\%options, "apps_uuid_suffix", $app);
if (defined $app_userid_suffix) {
$uuid =~ s/$app_userid_suffix(\s|\(|\[])*$//i;
}

$uuid =~ s/^(-|_|\s|\(|\[])+//;
$uuid =~ s/(-|_|\s|\)|\])+$//;
}

# If we have a uuid from an app, make the userid a combination of app + uuid
if ((defined $uuid) and ($uuid !~ /^(-|_|\s|-|_|\.)*$/)) {
$userid = $app . '.' . $uuid;
}
# otherwise use the original userid used for the API if any
elsif (defined $change_ref->{userid}) {
$userid = $change_ref->{userid};
}
}

if ((not defined $userid) or ($userid eq '')) {
if (not defined $userid) {
$userid = "openfoodfacts-contributors";
}

# Add the app to the change structure if we identified one, this will be used to populate the data sources field
if (defined $app) {
$change_ref->{app} = $app;
}

$log->debug("get_change_userid_or_uuid", { change_ref => $change_ref, app => $app, app_userid_prefix => $app_userid_prefix, uuid => $uuid, userid => $userid } ) if $log->is_debug();

return $userid;
}

Expand Down
3 changes: 3 additions & 0 deletions lib/ProductOpener/Tags.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1756,8 +1756,11 @@ sub generate_tags_taxonomy_extract ($$$$) {
my $options_ref = shift;
my $lcs_ref = shift;

$log->debug("generate_tags_taxonomy_extract", {tagtype => $tagtype, tags_ref => $tags_ref, options_ref => $options_ref, lcs_ref => $lcs_ref }) if $log->is_debug();

# Return empty hash if the taxonomy does not exist
if (not defined $translations_to{$tagtype}) {
$log->debug("taxonomy not found", {tagtype => $tagtype}) if $log->is_debug();
return {};
}

Expand Down
5 changes: 5 additions & 0 deletions po/common/common.pot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5960,3 +5960,8 @@ msgstr "This might not be the answer people want to hear, but there is no safe l
msgctxt "source"
msgid "Source"
msgstr "Source"

# variable names between { } must not be translated
msgctxt "f_app_user"
msgid "A user of the {app_name} app"
msgstr "A user of the {app_name} app"
5 changes: 5 additions & 0 deletions po/common/en.po
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5988,3 +5988,8 @@ msgstr "This might not be the answer people want to hear, but there is no safe l
msgctxt "source"
msgid "Source"
msgstr "Source"

# variable names between { } must not be translated
msgctxt "f_app_user"
msgid "A user of the {app_name} app"
msgstr "A user of the {app_name} app"
10 changes: 7 additions & 3 deletions scripts/update_all_products.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,7 @@
$product_ref->{rev} = $last_rev;
my $blame_ref = {};
compute_product_history_and_completeness($data_root, $product_ref, $changes_ref, $blame_ref);
compute_data_sources($product_ref);
compute_data_sources($product_ref, $changes_ref);
store("$data_root/products/$path/changes.sto", $changes_ref);
}
else {
Expand Down Expand Up @@ -831,7 +831,11 @@
}

if ($compute_data_sources) {
compute_data_sources($product_ref);
my $changes_ref = retrieve("$data_root/products/$path/changes.sto");
if (not defined $changes_ref) {
$changes_ref = [];
}
compute_data_sources($product_ref, $changes_ref);
}

if ($compute_nova) {
Expand Down Expand Up @@ -1016,7 +1020,7 @@
}
my $blame_ref = {};
compute_product_history_and_completeness($data_root, $product_ref, $changes_ref, $blame_ref);
compute_data_sources($product_ref);
compute_data_sources($product_ref, $changes_ref);
store("$data_root/products/$path/changes.sto", $changes_ref);
}

Expand Down
Loading