Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Radar plus bugfixes #32

Merged
merged 8 commits into from
Apr 19, 2023
Merged

Radar plus bugfixes #32

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c109857
Ajouter m$short_name dans la clé de frontpage
ClementCadieux Apr 18, 2023
28be0c5
Ajout d'une deuxième façon de scraper CTV
ClementCadieux Apr 18, 2023
6c1599d
Retry on pushing to lake if it fails
ClementCadieux Apr 18, 2023
0c9e8e4
Ajout champ metadata frontpage_root_key + fix typo
ClementCadieux Apr 18, 2023
176a477
Ajouter counter max et sleep time
ClementCadieux Apr 18, 2023
a58082a
Changer le nom de keys_unes pour headline_keys
ClementCadieux Apr 18, 2023
0ece76f
Enlever headline_keys
ClementCadieux Apr 18, 2023
288ef92
Rajouter headline_root_key
ClementCadieux Apr 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 70 additions & 35 deletions pipelines/extractors/e_radar+.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,14 @@ harvest_headline <- function(r, m) {
rvest::html_nodes("a") %>%
rvest::html_attr("href")

if(length(CTV_extracted_headline) == 0){
clessnverse::logit(scriptname, "CTV: Initial attempt failed, trying thorugh xpaths.", logger)
CTV_extracted_headline <- r %>%
rvest::html_nodes(xpath = '//div[@class="c-list__item__block"]') %>%
rvest::html_nodes(xpath = '//a[@class="c-list__item__image"]') %>%
rvest::html_attr("href")
}

if(length(CTV_extracted_headline) > 0){
if (grepl("^http.*", CTV_extracted_headline[[1]])) {
url <- CTV_extracted_headline[[1]]
Expand Down Expand Up @@ -377,7 +385,8 @@ harvest_headline <- function(r, m) {
storage_class = "lake",
country = m$country,
schema = opt$schema,
hashedHTML = NA
hashed_html = NA,
frontpage_root_key = NA
)

if (r$response$status_code == 200) {
Expand All @@ -397,25 +406,37 @@ harvest_headline <- function(r, m) {

pushedHeadlines <<- append(pushedHeadlines, key)

hub_response <- clessnverse::commit_lake_item(
data = list(
key = key,
path = "radarplus/headline",
item = doc
),
metadata = metadata,
mode = if (opt$refresh_data) "refresh" else "newonly",
credentials
)
pushed <- FALSE
counter <- 0

if (hub_response) {
clessnverse::logit(scriptname, paste("successfuly pushed headline", key, "to datalake"), logger)
nb_headline <<- nb_headline + 1
} else {
clessnverse::logit(scriptname, paste("error while pushing headline", key, "to datalake"), logger)
while(!pushed && counter < 20){
if(counter > 0){
Sys.sleep(20)
}
hub_response <- clessnverse::commit_lake_item(
data = list(
key = key,
path = "radarplus/headline",
item = doc
),
metadata = metadata,
mode = if (opt$refresh_data) "refresh" else "newonly",
credentials
)

if (hub_response) {
clessnverse::logit(scriptname, paste("successfuly pushed headline", key, "to datalake"), logger)
nb_headline <<- nb_headline + 1
pushed <- TRUE
} else {
clessnverse::logit(scriptname, paste("error while pushing headline", key, "to datalake"), logger)
counter <- counter + 1
}
}

if(!pushed){
warning(paste("error while pushing headline", key, "to datalake"))
}

} else {
clessnverse::logit(scriptname, paste("there was an error getting url", url), logger)
warning(paste("there was an error getting url", url))
Expand Down Expand Up @@ -452,7 +473,7 @@ main <- function() {
storage_class = "lake",
country = m$country,
schema = opt$schema,
keysUne = NA
headline_root_key = NA
)

r <<- rvest::session(url)
Expand All @@ -472,27 +493,41 @@ main <- function() {
keyUrl <- substr(keyUrl, 1, nchar(keyUrl) - 1)
}

key <- gsub(" |-|:|/|\\.", "_", paste(stringr::str_match(keyUrl, "[^/]+$"), Sys.time(), sep="_"))
key <- gsub(" |-|:|/|\\.", "_", paste(m$short_name, stringr::str_match(keyUrl, "[^/]+$"), Sys.time(), sep="_"))
if (opt$refresh_data) mode <- "refresh" else mode <- "newonly"

hub_response <- clessnverse::commit_lake_item(
data = list(
key = key,
path = "radarplus/frontpage",
item = doc
),
metadata = metadata,
mode = mode,
credentials = credentials
)
pushed <- FALSE
counter <- 0

while(!pushed){
if(counter > 0){
Sys.sleep(20)
}
hub_response <- clessnverse::commit_lake_item(
data = list(
key = key,
path = "radarplus/frontpage",
item = doc
),
metadata = metadata,
mode = mode,
credentials = credentials
)

if (hub_response) {
clessnverse::logit(scriptname, paste("successfuly pushed frontpage", key, "to datalake"), logger)
nb_frontpage <<- nb_frontpage + 1
pushed <- TRUE
} else {
clessnverse::logit(scriptname, paste("error while pushing frontpage", key, "to datalake"), logger)
counter <- counter + 1
}
}

if (hub_response) {
clessnverse::logit(scriptname, paste("successfuly pushed frontpage", key, "to datalake"), logger)
nb_frontpage <<- nb_frontpage + 1
harvest_headline(r, m)
if(pushed){
harvest_headline(r, m)
} else {
clessnverse::logit(scriptname, paste("error while pushing frontpage", key, "to datalake"), logger)
warning(paste("error while pushing frontpage", key, "to datalake"))
warning(paste("error while pushing frontpage", key, "to datalake"))
}

} else {
Expand Down