diff --git a/.env b/.env index f562afd47d64c..e5352638ac415 100644 --- a/.env +++ b/.env @@ -38,6 +38,7 @@ MONGO_INITDB_ROOT_USERNAME=root MONGO_INITDB_ROOT_PASSWORD=test ROBOTOFF_URL=http://robotoff.openfoodfacts.localhost:5500 # connect to Robotoff running in separate docker-compose deployment EVENTS_URL= +REDIS_URL=searchredis:6379 # Redis runs in a separate docker-compose deployment GOOGLE_CLOUD_VISION_API_KEY= CROWDIN_PROJECT_IDENTIFIER= CROWDIN_PROJECT_KEY= diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 3dd2e4a54514e..6ddbebd210a07 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -113,6 +113,7 @@ jobs: echo "POSTGRES_USER=${{ secrets.POSTGRES_USER }}" >> .env echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env echo "ROBOTOFF_URL=${{ secrets.ROBOTOFF_URL }}" >> .env + echo "REDIS_URL=${{ secrets.REDIS_URL }}" >> .env echo "GOOGLE_CLOUD_VISION_API_KEY=${{ secrets.GOOGLE_CLOUD_VISION_API_KEY }}" >> .env echo "CROWDIN_PROJECT_IDENTIFIER=${{ secrets.CROWDIN_PROJECT_IDENTIFIER }}" >> .env echo "CROWDIN_PROJECT_KEY=${{ secrets.CROWDIN_PROJECT_KEY }}" >> .env diff --git a/conf/apache.conf b/conf/apache.conf index cc27fee09a049..dfe979a8a2f51 100644 --- a/conf/apache.conf +++ b/conf/apache.conf @@ -17,6 +17,7 @@ PerlPassEnv ROBOTOFF_URL PerlPassEnv EVENTS_URL PerlPassEnv EVENTS_USERNAME PerlPassEnv EVENTS_PASSWORD +PerlPassEnv REDIS_URL PerlPassEnv MONGODB_HOST PerlPassEnv GOOGLE_CLOUD_VISION_API_KEY PerlPassEnv CROWDIN_PROJECT_IDENTIFIER diff --git a/cpanfile b/cpanfile index f04803ec70e98..250a92ea4c438 100644 --- a/cpanfile +++ b/cpanfile @@ -64,6 +64,7 @@ requires 'JSON::Create'; requires 'JSON::Parse'; requires 'Data::DeepAccess'; requires 'XML::XML2JSON'; +requires 'Redis::Client'; # Mojolicious/Minion diff --git a/docker-compose.yml b/docker-compose.yml index cabcf6e59724d..a40802e0ce481 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ x-backend-conf: &backend-conf - EVENTS_URL - EVENTS_USERNAME - EVENTS_PASSWORD + - REDIS_URL - GOOGLE_CLOUD_VISION_API_KEY - CROWDIN_PROJECT_IDENTIFIER - CROWDIN_PROJECT_KEY diff --git a/docs/introduction/dev-environment-quick-start-guide.md b/docs/introduction/dev-environment-quick-start-guide.md index e99d1b9a43f83..657302c738734 100644 --- a/docs/introduction/dev-environment-quick-start-guide.md +++ b/docs/introduction/dev-environment-quick-start-guide.md @@ -77,6 +77,7 @@ The `.env` file contains ProductOpener default settings: | `PRODUCT_OPENER_FLAVOR_SHORT` | can be modified to run different flavors of OpenFoodFacts, amongst `off` (default), `obf`, `oppf`, `opf`.| | `PRODUCERS_PLATFORM` | can be set to `1` to build / run the **producer platform**.| | `ROBOTOFF_URL` | can be set to **connect with a Robotoff instance**.| +| `REDIS_URL` | can be set to **connect with a Redis instance for populating the search index**.| | `GOOGLE_CLOUD_VISION_API_KEY` | can be set to **enable OCR using Google Cloud Vision**.| | `CROWDIN_PROJECT_IDENTIFIER` and `CROWDIN_PROJECT_KEY` | can be set to **run translations**.| | `GEOLITE2_PATH`, `GEOLITE2_ACCOUNT_ID` and `GEOLITE2_LICENSE_KEY` | can be set to **enable Geolite2**.| diff --git a/lib/ProductOpener/Config2_docker.pm b/lib/ProductOpener/Config2_docker.pm index 5b3184923d3f6..0a8c06ddfc68b 100755 --- a/lib/ProductOpener/Config2_docker.pm +++ b/lib/ProductOpener/Config2_docker.pm @@ -49,6 +49,7 @@ BEGIN $events_url $events_username $events_password + $redis_url %server_options ); %EXPORT_TAGS = (all => [@EXPORT_OK]); @@ -102,6 +103,9 @@ $events_url = $ENV{EVENTS_URL}; $events_username = $ENV{EVENTS_USERNAME}; $events_password = $ENV{EVENTS_PASSWORD}; +# Set this to your instance of the search service to enable writes to it +$redis_url = $ENV{REDIS_URL}; + %server_options = ( private_products => $producers_platform, # 1 to make products visible only to the owner (producer platform) producers_platform => $producers_platform, diff --git a/lib/ProductOpener/Config2_sample.pm b/lib/ProductOpener/Config2_sample.pm index abd7fbdc0a0a0..b02c83327be67 100644 --- a/lib/ProductOpener/Config2_sample.pm +++ b/lib/ProductOpener/Config2_sample.pm @@ -43,6 +43,7 @@ BEGIN $events_url $events_username $events_password + $redis_url %server_options ); @@ -82,6 +83,8 @@ $events_url = ''; $events_username = ''; $events_password = ''; +$redis_url = ''; + %server_options = ( cookie_domain => "openfoodfacts.dev", # if not set, default to $server _domain diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm index 36fb99711fc06..ad195600e19fd 100644 --- a/lib/ProductOpener/Products.pm +++ b/lib/ProductOpener/Products.pm @@ -133,6 +133,7 @@ use ProductOpener::Data qw/:all/; use ProductOpener::MainCountries qw/:all/; use ProductOpener::Text qw/:all/; use ProductOpener::Display qw/single_param/; +use ProductOpener::Redis qw/push_to_search_service/; use CGI qw/:cgi :form escapeHTML/; use Encode; @@ -1178,6 +1179,9 @@ sub store_product($user_id, $product_ref, $comment) { store("$new_data_root/products/$path/changes.sto", $changes_ref); log_change($product_ref, $change_ref); + # index for search service + push_to_search_service($product_ref); + $log->debug("store_product - done", { code => $code, product_id => $product_id } ) if $log->is_debug(); return 1; diff --git a/lib/ProductOpener/Redis.pm b/lib/ProductOpener/Redis.pm new file mode 100644 index 0000000000000..9b2977ebd5ff9 --- /dev/null +++ b/lib/ProductOpener/Redis.pm @@ -0,0 +1,52 @@ + +package ProductOpener::Redis; + +use ProductOpener::PerlStandards; +use Exporter qw< import >; + +BEGIN { + use vars qw(@ISA @EXPORT_OK %EXPORT_TAGS); + @EXPORT_OK = qw( + &push_to_search_service + ); # symbols to export on request + %EXPORT_TAGS = (all => [@EXPORT_OK]); +} + +use vars @EXPORT_OK; + +use Log::Any qw($log); +use ProductOpener::Config2; +use Redis::Client; + +sub init_redis() { + + $log->debug("init_redis", {redis_url_env => $ENV{REDIS_URL}, redis_url => $ProductOpener::Config2::redis_url}) + if $log->is_debug(); + + $log->warn("REDIS_URL env", {redis_url => $ENV{REDIS_URL}}) if $log->is_warn(); + if ($ProductOpener::Config2::redis_url eq "") { + $log->warn("Redis URL not provided for search indexing", {error => $@}) if $log->is_warn(); + return; + } + my $redis_client; + eval {$redis_client = Redis::Client->new(host => $ProductOpener::Config2::redis_url);}; + if ($@) { + $log->warn("Error connecting to Redis", {error => $@}) if $log->is_warn(); + } + else { + return $redis_client; + } + return; +} + +my $redis_client = init_redis(); + +sub push_to_search_service ($product_ref) { + if (defined($redis_client)) { + eval {$redis_client->rpush('search_import_queue', $product_ref->{code});}; + } + + return; +} + +1;