feat: replacing incron with a specific script (#8234)

Replaces incron with a specific script.

Fixes:
- #8209 incron does not exists in Debian Bullseye

Dockerfile

@@ -8,11 +8,12 @@ ARG CPANMOPTS=
 ######################
 # Base modperl image stage
 ######################
-FROM bitnami/minideb:buster AS modperl
+FROM debian:bullseye AS modperl
 
 # Install cpm to install cpanfile dependencies
 RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt set -x && \
-    install_packages \
+    apt update && \
+    apt install -y \
         apache2 \
         apt-utils \
         cpanminus \
@@ -73,6 +74,8 @@ RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt set -x && \
         #
         # Action::Retry
         libmath-fibonacci-perl \
+        # EV - event loop
+        libev-perl \
         # Algorithm::CheckDigits
         libprobe-perl-perl \
         # CLDR::Number
@@ -137,7 +140,6 @@ RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt set -x && \
         libtest-number-delta-perl \
         libdevel-size-perl \
         gnumeric \
-        incron \
         # for dev
         # gnu readline
         libreadline-dev \
@@ -203,10 +205,6 @@ RUN \
     chown www-data:www-data -R /var/log
 # Install Product Opener from the workdir
 COPY --chown=www-data:www-data . /opt/product-opener/
-RUN \
-    # www-data user shall be able to use incron
-    echo www-data >> /etc/incron.allow && \
-    incrontab -u www-data /opt/product-opener/conf/incron.conf
 
 EXPOSE 80
 COPY ./docker/docker-entrypoint.sh /

Makefile

@@ -39,7 +39,8 @@ HOSTS=127.0.0.1 world.productopener.localhost fr.productopener.localhost static.
 DOCKER_COMPOSE=docker-compose --env-file=${ENV_FILE}
 # we run tests in a specific project name to be separated from dev instances
 # we also publish mongodb on a separate port to avoid conflicts
-DOCKER_COMPOSE_TEST=COMPOSE_PROJECT_NAME=po_test PO_COMMON_PREFIX=test_ MONGO_EXPOSE_PORT=27027 docker-compose --env-file=${ENV_FILE}
+# we also enable the possibility to fake services in po_test_runner
+DOCKER_COMPOSE_TEST=ROBOTOFF_URL="http://backend:8881/" GOOGLE_CLOUD_VISION_API_URL="http://backend:8881/" COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}_test PO_COMMON_PREFIX=test_ MONGO_EXPOSE_PORT=27027 docker-compose --env-file=${ENV_FILE}
 
 .DEFAULT_GOAL := dev
 
@@ -238,7 +239,8 @@ integration_test:
 	@echo "🥫 Running unit tests …"
 # we launch the server and run tests within same container
 # we also need dynamicfront for some assets to exists
-	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront
+# this is the place where variables are important
+	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront incron
 # note: we need the -T option for ci (non tty environment)
 	${DOCKER_COMPOSE_TEST} exec -T backend prove -l -r tests/integration
 	${DOCKER_COMPOSE_TEST} stop
@@ -258,7 +260,7 @@ test-unit: guard-test
 # usage:  make test-int test=test-name.t
 test-int: guard-test # usage: make test-one test=test-file.t
 	@echo "🥫 Running test: 'tests/integration/${test}' …"
-	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront
+	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront incron
 	${DOCKER_COMPOSE_TEST} exec backend perl tests/integration/${test}
 # better shutdown, for if we do a modification of the code, we need a restart
 	${DOCKER_COMPOSE_TEST} stop backend
@@ -267,9 +269,13 @@ test-int: guard-test # usage: make test-one test=test-file.t
 stop_tests:
 	${DOCKER_COMPOSE_TEST} stop
 
+# clean tests, remove containers and volume (useful if you changed env variables, etc.)
+clean_tests:
+	${DOCKER_COMPOSE_TEST} down -v --remove-orphans
+
 update_tests_results:
 	@echo "🥫 Updated expected test results with actuals for easy Git diff"
-	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront
+	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront incron
 	${DOCKER_COMPOSE_TEST} exec -T -w /opt/product-opener/tests backend bash update_tests_results.sh
 	${DOCKER_COMPOSE_TEST} stop
 

conf/apache.conf

@@ -21,6 +21,7 @@ PerlPassEnv EVENTS_PASSWORD
 PerlPassEnv REDIS_URL
 PerlPassEnv MONGODB_HOST
 PerlPassEnv GOOGLE_CLOUD_VISION_API_KEY
+PerlPassEnv GOOGLE_CLOUD_VISION_API_URL
 PerlPassEnv CROWDIN_PROJECT_IDENTIFIER
 PerlPassEnv CROWDIN_PROJECT_KEY
 PerlPassEnv GEOLITE2_PATH

cpanfile

@@ -84,6 +84,10 @@ requires 'Log::Any::Adapter::Log4perl', '>= 0.09'; # liblog-any-adapter-log4perl
 requires 'Action::CircuitBreaker';
 requires 'Action::Retry'; # deps: libmath-fibonacci-perl
 
+# AnyEvent
+requires 'AnyEvent';
+requires 'AnyEvent::Inotify::Simple';
+
 on 'test' => sub {
   requires 'Test::More', '>= 1.302186, < 2.0';
   requires 'Test::MockModule';
@@ -100,6 +104,8 @@ on 'test' => sub {
   requires 'Devel::Cover';
   requires 'Devel::Cover::Report::Codecov';
   requires 'Devel::Cover::Report::Codecovbash';
+  requires 'Test::Fake::HTTPD';
+  requires 'URL::Encode';
 };
 
 on 'develop' => sub {

docker-compose.yml

@@ -20,6 +20,7 @@ x-backend-conf: &backend-conf
     - EVENTS_PASSWORD
     - REDIS_URL
     - GOOGLE_CLOUD_VISION_API_KEY
+    - GOOGLE_CLOUD_VISION_API_URL
     - CROWDIN_PROJECT_IDENTIFIER
     - CROWDIN_PROJECT_KEY
     - GEOLITE2_PATH
@@ -85,10 +86,8 @@ services:
       - "$MINION_QUEUE"
   incron:
     <<: *backend-conf
-    # This service run the incron jobs
-    # Only root can run incron
-    user: root
-    command: ["incrond", "-n"]
+    # This service watch for new images to trigger ocr and robotoff processing
+    command: ["perl", "scripts/run_cloud_vision_ocr.pl", "/mnt/podata/new_images"]
   frontend:
     image: ghcr.io/openfoodfacts/openfoodfacts-server/frontend:${TAG}
     depends_on:

lib/ProductOpener/APITest.pm

@@ -52,6 +52,7 @@ BEGIN {
 		&wait_dynamic_front
 		&execute_api_tests
 		&wait_server
+		&fake_http_server
 	);    # symbols to export on request
 	%EXPORT_TAGS = (all => [@EXPORT_OK]);
 }
@@ -61,6 +62,7 @@ use vars @EXPORT_OK;
 use ProductOpener::TestDefaults qw/:all/;
 use ProductOpener::Test qw/:all/;
 use ProductOpener::Mail qw/ $LOG_EMAIL_START $LOG_EMAIL_END /;
+use ProductOpener::Store qw/store retrieve/;
 
 use Test::More;
 use LWP::UserAgent;
@@ -71,6 +73,7 @@ use JSON::PP;
 use Carp qw/confess/;
 use Clone qw/clone/;
 use File::Tail;
+use Test::Fake::HTTPD;
 
 # Constants of the test website main domain and url
 # Should be used internally only (see: construct_test_url to build urls in tests)
@@ -153,6 +156,8 @@ Return a user agent
 sub new_client () {
 	my $jar = HTTP::CookieJar::LWP->new;
 	my $ua = LWP::UserAgent->new(cookie_jar => $jar);
+	# set a neutral user-agent, for it may appear in some results
+	$ua->agent("Product-opener-tests/1.0");
 	return $ua;
 }
 
@@ -594,7 +599,7 @@ Especially we replace "3D=" for "=" and join line and their continuation
 =head4 $mail text of mail
 
 =head3 Returns
-Reformated text
+Reformatted text
 =cut
 
 sub mail_to_text ($mail) {
@@ -620,10 +625,11 @@ ref to an array of lines of the email
 
 sub normalize_mail_for_comparison ($mail) {
 	# remove boundaries
+	$DB::single = 1;
 	my $text = mail_to_text($mail);
-	my @boundaries = $text =~ m/boundary="([^"]+)"/g;
+	my @boundaries = $text =~ m/boundary=([^ ,\n\t]+)/g;
 	foreach my $boundary (@boundaries) {
-		$text =~ s/$boundary/\\"--boundary--\\"/g;
+		$text =~ s/$boundary/boundary/g;
 	}
 	# replace generic dates
 	$text =~ s/\d\d\d\d-\d\d-\d\d/--date--/g;
@@ -634,4 +640,74 @@ sub normalize_mail_for_comparison ($mail) {
 	return \@lines;
 }
 
+=head2 fake_http_server($port, $dump_path, $responses_ref) {
+
+Launch a fake HTTP server.
+
+We use that to simulate Robotoff or any HTTP API in integration tests.
+As it will be launched on the local backend container, we have to pretend
+those service URL is on C<backend:$port>.
+
+You can provide a list of responses to simulate real service responses,
+while requests sent are store for later checks by the tests.
+
+=head3 parameters
+
+=head4 $dump_path - path
+
+A temporary directory to dump requests
+
+You can retrieve requests, in this directory as C<req-n.sto>
+
+=head4 $responses_ref - ref to a list
+
+List of responses to send, in right order, for each received request.
+
+If the number of request exceed this list,
+we will send simple 200 HTTP responses with a json payload.
+
+=head3 returns ref to fake server
+
+Hold the reference until you don't need the server
+
+=cut
+
+sub fake_http_server ($port, $dump_path, $responses_ref) {
+
+	# dump responses
+	my $resp_num = 0;
+	foreach my $resp (@$responses_ref) {
+		store("$dump_path/resp-$resp_num.sto", $resp);
+		$resp_num += 1;
+	}
+
+	my $httpd = Test::Fake::HTTPD->new(
+		timeout => 1000,
+		listen => 10,
+		host => "0.0.0.0",
+		port => $port,
+	);
+
+	$httpd->run(
+		sub {
+			my $req = shift;
+			my @dumped_reqs = glob("$dump_path/req-*.sto");
+			my $num_req = scalar @dumped_reqs;
+			# dump request to the folder
+			store("$dump_path/req-$num_req.sto", $req);
+			# look for an eventual response
+			my $response_ref;
+			if (-e "$dump_path/resp-$num_req.sto") {
+				$response_ref = retrieve("$dump_path/resp-$num_req.sto");
+			}
+			else {
+				# an ok response
+				$response_ref = HTTP::Response->new("200", "OK", HTTP::Headers->new(), '{"foo": "blah"}');
+			}
+			return $response_ref;
+		}
+	);
+	return $httpd;
+}
+
 1;

lib/ProductOpener/Config2_docker.pm

@@ -43,6 +43,7 @@ BEGIN {
 		$mongodb_timeout_ms
 		$memd_servers
 		$google_cloud_vision_api_key
+		$google_cloud_vision_api_url
 		$crowdin_project_identifier
 		$crowdin_project_key
 		$robotoff_url
@@ -86,6 +87,8 @@ $mongodb_timeout_ms = 50000;    # config option max_time_ms/maxTimeMS
 $memd_servers = ["memcached:11211"];
 
 $google_cloud_vision_api_key = $ENV{GOOGLE_CLOUD_VISION_API_KEY};
+$google_cloud_vision_api_url = $ENV{GOOGLE_CLOUD_VISION_API_URL} || "https://vision.googleapis.com/v1/images:annotate";
+
 $crowdin_project_identifier = $ENV{CROWDIN_PROJECT_IDENTIFIER};
 $crowdin_project_key = $ENV{CROWDIN_PROJECT_KEY};
 

lib/ProductOpener/Config_obf.pm

@@ -43,6 +43,7 @@ BEGIN {
 		$producers_email
 
 		$google_cloud_vision_api_key
+		$google_cloud_vision_api_url
 
 		$crowdin_project_identifier
 		$crowdin_project_key
@@ -196,6 +197,7 @@ $conf_root = $ProductOpener::Config2::conf_root;
 $geolite2_path = $ProductOpener::Config2::geolite2_path;
 
 $google_cloud_vision_api_key = $ProductOpener::Config2::google_cloud_vision_api_key;
+$google_cloud_vision_api_url = $ProductOpener::Config2::google_cloud_vision_api_url;
 
 $crowdin_project_identifier = $ProductOpener::Config2::crowdin_project_identifier;
 $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;

lib/ProductOpener/Config_off.pm

@@ -42,6 +42,7 @@ BEGIN {
 		$producers_email
 
 		$google_cloud_vision_api_key
+		$google_cloud_vision_api_url
 
 		$crowdin_project_identifier
 		$crowdin_project_key
@@ -343,6 +344,7 @@ $conf_root = $ProductOpener::Config2::conf_root;
 $geolite2_path = $ProductOpener::Config2::geolite2_path;
 
 $google_cloud_vision_api_key = $ProductOpener::Config2::google_cloud_vision_api_key;
+$google_cloud_vision_api_url = $ProductOpener::Config2::google_cloud_vision_api_url;
 
 $crowdin_project_identifier = $ProductOpener::Config2::crowdin_project_identifier;
 $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;

lib/ProductOpener/Config_opf.pm

@@ -43,6 +43,7 @@ BEGIN {
 		$producers_email
 
 		$google_cloud_vision_api_key
+		$google_cloud_vision_api_url
 
 		$crowdin_project_identifier
 		$crowdin_project_key
@@ -194,6 +195,7 @@ $conf_root = $ProductOpener::Config2::conf_root;
 $geolite2_path = $ProductOpener::Config2::geolite2_path;
 
 $google_cloud_vision_api_key = $ProductOpener::Config2::google_cloud_vision_api_key;
+$google_cloud_vision_api_url = $ProductOpener::Config2::google_cloud_vision_api_url;
 
 $crowdin_project_identifier = $ProductOpener::Config2::crowdin_project_identifier;
 $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;

lib/ProductOpener/Config_opff.pm

@@ -43,6 +43,7 @@ BEGIN {
 		$producers_email
 
 		$google_cloud_vision_api_key
+		$google_cloud_vision_api_url
 
 		$crowdin_project_identifier
 		$crowdin_project_key
@@ -193,6 +194,7 @@ $conf_root = $ProductOpener::Config2::conf_root;
 $geolite2_path = $ProductOpener::Config2::geolite2_path;
 
 $google_cloud_vision_api_key = $ProductOpener::Config2::google_cloud_vision_api_key;
+$google_cloud_vision_api_url = $ProductOpener::Config2::google_cloud_vision_api_url;
 
 $crowdin_project_identifier = $ProductOpener::Config2::crowdin_project_identifier;
 $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;