From 9df8dab744570a19d04626aab8f621e84812f642 Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Thu, 6 Sep 2018 18:51:59 -0300 Subject: [PATCH 1/8] Veracity integration --- .env.example | 9 ++ app/Http/Controllers/VeracityController.php | 27 ++++++ app/Jobs/RunVeracityService.php | 95 +++++++++++++++++++++ config/options.php | 9 ++ routes/web.php | 3 +- 5 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 app/Http/Controllers/VeracityController.php create mode 100644 app/Jobs/RunVeracityService.php diff --git a/.env.example b/.env.example index e4770da..572ef97 100644 --- a/.env.example +++ b/.env.example @@ -39,3 +39,12 @@ ACTIONABILITY_REQUEST_COST= ACTIONABILITY_QUOTA_RESET= ACTIONABILITY_API_USERNAME= ACTIONABILITY_API_PASSWORD= + + + +VERACITY_API_URL= +VERACITY_QUOTA= +VERACITY_REQUEST_COST= +VERACITY_QUOTA_RESET= +VERACITY_API_USERNAME= +VERACITY_API_PASSWORD= diff --git a/app/Http/Controllers/VeracityController.php b/app/Http/Controllers/VeracityController.php new file mode 100644 index 0000000..2e7e8f1 --- /dev/null +++ b/app/Http/Controllers/VeracityController.php @@ -0,0 +1,27 @@ +all(); + dispatch(new RunVeracityService($post)); + return; + } + + public function all(Request $request) + { + $post = $request->all(); + dispatch(new RunVeracityService($post, 'all')); + + return; + } +} diff --git a/app/Jobs/RunVeracityService.php b/app/Jobs/RunVeracityService.php new file mode 100644 index 0000000..c03d33f --- /dev/null +++ b/app/Jobs/RunVeracityService.php @@ -0,0 +1,95 @@ + 'ushahidi', + 'route' => 'veracity', + ], $remaining, $minutes); + + if (!$throttler->check()) { + $this->release($time); + return; + } + $this->runService($this->post); + } + + public function requestProcessing($text) + { + try { + $client = new Client(); + $url = config('options.veracity.api.url'); + $response = $client->request('POST', $url, + [ + 'headers' => [ + 'Accept' => 'application/json', + 'Content-type' => 'text/plain', + ], + 'auth' => [ + config('options.veracity.api.username'), + config('options.veracity.api.password') + ], + 'body' => json_encode([$text]) + ] + ); + return $response; + } catch (RequestException $e) { + if ($e->hasResponse()) { + Log::error(Psr7\str($e->getResponse())); + } + } + } + + public function format_as_post($post, $response) + { + + $tags = []; + $json = json_decode($response->getBody(), true); + switch($json['rumour_label']) { + case 'unverified': + $label = 'Unverified'; + break; + case false: + $label ='Not a rumor'; + break; + default: + $label = 'Rumor'; + break; + } + $tags = array_merge($tags, [$label]); + // At the moment it is important to only set fields that you intend to change + // as Post updates are async it is possible to overwrite other user's data + // by performing a full update with the complete object recevied + return [ + 'id' => $post['id'], + 'tags' => $tags, + 'values' => [], + 'webhook_uuid' => $post['webhook_uuid'] + ]; + } +} diff --git a/config/options.php b/config/options.php index bdbc142..9870e0d 100644 --- a/config/options.php +++ b/config/options.php @@ -30,5 +30,14 @@ ], 'request_per_time_block' => env('ACTIONABILITY_QUOTA') / env('ACTIONABILITY_REQUEST_COST'), 'quota_reset' => env('ACTIONABILITY_QUOTA_RESET') + ], + 'veracity' => [ + 'api' => [ + 'url' => env('VERACITY_API_URL'), + 'username' => env('VERACITY_API_USERNAME'), + 'password' => env('VERACITY_API_PASSWORD') + ], + 'request_per_time_block' => env('VERACITY_QUOTA') / env('VERACITY_REQUEST_COST'), + 'quota_reset' => env('VERACITY_QUOTA_RESET') ] ]; diff --git a/routes/web.php b/routes/web.php index d3dacfd..ea9de69 100644 --- a/routes/web.php +++ b/routes/web.php @@ -22,4 +22,5 @@ $app->post('/crees/eventType', ['middleware' => 'UshahidiRequestValidator', 'uses' => 'CreesController@eventType']); $app->post('/crees/infoType', ['middleware' => 'UshahidiRequestValidator', 'uses' => 'CreesController@infoType']); -$app->post('/actionability/annotate', ['middleware' => 'UshahidiRequestValidator', 'uses' => 'ActionabilityController@annotate']); \ No newline at end of file +$app->post('/actionability/annotate', ['middleware' => 'UshahidiRequestValidator', 'uses' => 'ActionabilityController@annotate']); +$app->post('/veracity/annotate', ['middleware' => 'UshahidiRequestValidator', 'uses' => 'VeracityController@annotate']); \ No newline at end of file From 34deefb7329bafe4b90627221ab4f2d26c3f1e3a Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Sun, 16 Sep 2018 15:28:59 -0300 Subject: [PATCH 2/8] Veracity scores --- app/Jobs/RunVeracityService.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Jobs/RunVeracityService.php b/app/Jobs/RunVeracityService.php index c03d33f..894797f 100644 --- a/app/Jobs/RunVeracityService.php +++ b/app/Jobs/RunVeracityService.php @@ -67,7 +67,6 @@ public function requestProcessing($text) public function format_as_post($post, $response) { - $tags = []; $json = json_decode($response->getBody(), true); switch($json['rumour_label']) { @@ -81,7 +80,7 @@ public function format_as_post($post, $response) $label = 'Rumor'; break; } - $tags = array_merge($tags, [$label]); + $tags = array_merge($tags, ['value' => $label, 'confidence_score' => $json['confidence'] * 100]); // At the moment it is important to only set fields that you intend to change // as Post updates are async it is possible to overwrite other user's data // by performing a full update with the complete object recevied From fbe59dc85e402dabfe0c8ebe8bd1ff86ea1f921f Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Sat, 22 Sep 2018 15:06:18 -0300 Subject: [PATCH 3/8] Change call to format_as_post for YODIE --- app/Jobs/RunProxyService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Jobs/RunProxyService.php b/app/Jobs/RunProxyService.php index 9becb79..4938d52 100644 --- a/app/Jobs/RunProxyService.php +++ b/app/Jobs/RunProxyService.php @@ -52,7 +52,7 @@ public function runService($post) $source_field_key = $post['source_field_key']; $response = $this->requestProcessing(array_get($post, $source_field_key)); - $post = $this->format_as_post($post, $response); + $post = $this->format_as_post($post, $response, array_get($post, $source_field_key)); dispatch(new UpdateUshahidiPost($post)); } From 819e7bd9086c53e5e078c67103ad9a66ffe99dae Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Sat, 22 Sep 2018 15:07:36 -0300 Subject: [PATCH 4/8] Change format_as_post to fix Yodie double wrapped text --- app/Jobs/RunYodieService.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Jobs/RunYodieService.php b/app/Jobs/RunYodieService.php index 9cbf74f..b2f40f5 100644 --- a/app/Jobs/RunYodieService.php +++ b/app/Jobs/RunYodieService.php @@ -81,7 +81,7 @@ public function replaceTextWithMD($indices, $link, $text) ); } - public function format_as_post($post, $response) + public function format_as_post($post, $response, $source_field=null) { $json = json_decode($response->getBody()); $yodie_post_field = config('options.ushahidi.survey_destination_field'); @@ -112,10 +112,10 @@ public function format_as_post($post, $response) foreach($replacement_strings as $replace => $value) { - $text = str_replace($replace, $value, $text); + $source_field = str_replace($replace, $value, $source_field); } - $yodie_text = $text; + $yodie_text = $source_field; $destination_field_key = $post['destination_field_key']; From 80b92adc3d47250f81e4363a25d93f4fe78640d6 Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Mon, 24 Sep 2018 13:56:27 -0300 Subject: [PATCH 5/8] Fix CREES tags - we were always sending the first character of the instead of the text of the post --- app/Jobs/RunCreesService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Jobs/RunCreesService.php b/app/Jobs/RunCreesService.php index 8119ee3..e3c14ac 100644 --- a/app/Jobs/RunCreesService.php +++ b/app/Jobs/RunCreesService.php @@ -55,7 +55,7 @@ public function requestProcessing($text) foreach ($crees_uris as $uri) { $response = $client->request('GET', $uri, [ - 'query' => ['text' => $text[0]] + 'query' => ['text' => $text] ] ); array_push($crees_reponses, $response); From 28a0671bc67e7ec39e61435dd38deba334f5c9db Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Fri, 28 Sep 2018 09:57:45 -0300 Subject: [PATCH 6/8] Refactored values to be able to use the same structure in all services with categories, including Veracity scores or not --- app/Jobs/RunActionabilityService.php | 6 +++--- app/Jobs/RunCreesService.php | 2 +- app/Jobs/RunVeracityService.php | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/Jobs/RunActionabilityService.php b/app/Jobs/RunActionabilityService.php index 9104ce3..17ccb66 100644 --- a/app/Jobs/RunActionabilityService.php +++ b/app/Jobs/RunActionabilityService.php @@ -71,11 +71,11 @@ public function format_as_post($post, $response) $json = json_decode($response->getBody(), true); if ($json['informative'] == false) { - $tags = array_merge($tags, ['Not informative']); + array_push($tags, ['value' => 'Not informative']); } else { - $tags = array_merge($tags, ['Informative']); + array_push($tags, ['value' => 'Informative']); foreach ($json['action_categories'] as $category) { - array_push($tags, $category['desc']); + array_push($tags, ['value' => $category['desc']]); } } // At the moment it is important to only set fields that you intend to change diff --git a/app/Jobs/RunCreesService.php b/app/Jobs/RunCreesService.php index e3c14ac..21b35b3 100644 --- a/app/Jobs/RunCreesService.php +++ b/app/Jobs/RunCreesService.php @@ -80,7 +80,7 @@ public function format_as_post($post, $responses) foreach ($responses as $response) { $json = json_decode($response->getBody()); $text = $json->label; - $tags = $tags ? array_merge($tags, [$text]) : [$text]; + array_push($tags, ['value' => $text]); } // At the moment it is important to only set fields that you intend to change diff --git a/app/Jobs/RunVeracityService.php b/app/Jobs/RunVeracityService.php index 894797f..705f60e 100644 --- a/app/Jobs/RunVeracityService.php +++ b/app/Jobs/RunVeracityService.php @@ -67,7 +67,6 @@ public function requestProcessing($text) public function format_as_post($post, $response) { - $tags = []; $json = json_decode($response->getBody(), true); switch($json['rumour_label']) { case 'unverified': @@ -80,7 +79,9 @@ public function format_as_post($post, $response) $label = 'Rumor'; break; } - $tags = array_merge($tags, ['value' => $label, 'confidence_score' => $json['confidence'] * 100]); + $tags = [ + ['value' => $label, 'confidence_score' => $json['confidence'] * 100] + ]; // At the moment it is important to only set fields that you intend to change // as Post updates are async it is possible to overwrite other user's data // by performing a full update with the complete object recevied From 0cc4264750a9e387e6bb2ecbfcfc73080e571f2a Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Mon, 3 Dec 2018 18:41:34 -0300 Subject: [PATCH 7/8] Switch Rumor and Not a Rumor --- app/Jobs/RunVeracityService.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Jobs/RunVeracityService.php b/app/Jobs/RunVeracityService.php index 705f60e..04e4179 100644 --- a/app/Jobs/RunVeracityService.php +++ b/app/Jobs/RunVeracityService.php @@ -73,10 +73,10 @@ public function format_as_post($post, $response) $label = 'Unverified'; break; case false: - $label ='Not a rumor'; + $label = 'Rumor'; break; default: - $label = 'Rumor'; + $label ='Not a rumor'; break; } $tags = [ From 9f443f3f31895d99d33f912f3d09ed182b5ff749 Mon Sep 17 00:00:00 2001 From: Romina Suarez Date: Wed, 19 Dec 2018 09:55:43 -0300 Subject: [PATCH 8/8] Switch changed to if for consistency --- app/Jobs/RunVeracityService.php | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/app/Jobs/RunVeracityService.php b/app/Jobs/RunVeracityService.php index 04e4179..7ce8d04 100644 --- a/app/Jobs/RunVeracityService.php +++ b/app/Jobs/RunVeracityService.php @@ -68,16 +68,12 @@ public function requestProcessing($text) public function format_as_post($post, $response) { $json = json_decode($response->getBody(), true); - switch($json['rumour_label']) { - case 'unverified': - $label = 'Unverified'; - break; - case false: - $label = 'Rumor'; - break; - default: - $label ='Not a rumor'; - break; + if ($json['rumour_label'] == 'unverified') { + $label = 'Unverified'; + } else if ($json['rumour_label'] == false) { + $label = 'Rumor'; + } else { + $label ='Not a rumor'; } $tags = [ ['value' => $label, 'confidence_score' => $json['confidence'] * 100]