diff --git a/R/getCOPDataFromDATIM.R b/R/getCOPDataFromDATIM.R index 6cdd64bb..7e8d5577 100644 --- a/R/getCOPDataFromDATIM.R +++ b/R/getCOPDataFromDATIM.R @@ -18,7 +18,7 @@ getCOPDataFromDATIM <- function(country_uids, inherits = TRUE)) { - if (!cop_year %in% c(2023:2024)) { + if (!cop_year %in% c(2023:2025)) { stop("The COP year provided is not supported by the internal function getCOPDataFromDATIM") ### NOTE for COP23 some special handling of SUBNAT data for FY23 like the code below may be diff --git a/R/getHTSModality.R b/R/getHTSModality.R index 5323abf0..d2051ddc 100644 --- a/R/getHTSModality.R +++ b/R/getHTSModality.R @@ -10,8 +10,10 @@ getHTSModality <- function(cop_year = getCurrentCOPYear(), d2_session = dynGet("d2_default_session", inherits = TRUE)) { + #Found here https://www.datim.org/dhis-web-maintenance/index.html#/list/dataElementSection/dataElementGroupSet groupSet <- switch( as.character(cop_year), + "2025" = "Bm4JmNS8ciD", #Need new one created in Datim touch base with Christian "2024" = "Bm4JmNS8ciD", "2023" = "fmxSIyzexmb", ) diff --git a/R/packTool.R b/R/packTool.R index 630307f5..e81dc2b3 100644 --- a/R/packTool.R +++ b/R/packTool.R @@ -88,7 +88,7 @@ packTool <- function(model_data = NULL, # Save & Export Workbook #### interactive_print("Saving...") - if (d$info$cop_year %in% c(2023, 2024) && d$info$tool == "Data Pack") { + if (d$info$cop_year %in% c(2023, 2024, 2025) && d$info$tool == "Data Pack") { tool_name <- "Target Setting Tool" } else { tool_name <- d$info$tool diff --git a/R/unPackTool.R b/R/unPackTool.R index 58db9fe5..1b927e8b 100644 --- a/R/unPackTool.R +++ b/R/unPackTool.R @@ -35,7 +35,7 @@ unPackTool <- function(submission_path = NULL, d <- unPackDataPack(d, d2_session = d2_session) - if (d$info$cop_year %in% c("2023", "2024")) { + if (d$info$cop_year %in% c("2023", "2024", "2025")) { d <- unpackYear2Sheet(d) } diff --git a/R/unPackingChecks.R b/R/unPackingChecks.R index a1a76c7f..934194cc 100644 --- a/R/unPackingChecks.R +++ b/R/unPackingChecks.R @@ -758,7 +758,8 @@ checkInvalidOrgUnits <- function(sheets, d, quiet = TRUE) { #There may be some variation in the columns between cop years cols_to_filter <- switch(as.character(d$info$cop_year), "2023" = c("PSNU", "Age", "Sex"), - "2024" = c("PSNU", "Age", "Sex")) + "2024" = c("PSNU", "Age", "Sex"), + "2025" = c("PSNU", "Age", "Sex")) invalid_orgunits <- d$sheets[sheets] %>% dplyr::bind_rows(.id = "sheet_name") %>% diff --git a/R/update_de_coc_co_map.R b/R/update_de_coc_co_map.R index 24bdd8a4..bfb715fa 100644 --- a/R/update_de_coc_co_map.R +++ b/R/update_de_coc_co_map.R @@ -48,15 +48,20 @@ update_de_coc_co_map <- function(cop_year = NULL, "dA9C5bL44NX", "FY24 MER Targets", 2024, "targets", "mer", "psnu", "lHUEzkjkij1", "FY25 MER Targets", 2025, "targets", "mer", "psnu", "lHUEzkjkij1", "FY26 MER Targets", 2026, "targets", "mer", "psnu", + "lHUEzkjkij1", "FY27 MER Targets", 2027, "targets", "mer", "psnu", #Update in November 2024 "vpDd67HlZcT", "FY24 DREAMS Targets", 2024, "targets", "dreams", "dsnu", "tNbhYbrKbnk", "FY25 DREAMS Targets", 2025, "targets", "dreams", "dsnu", "tNbhYbrKbnk", "FY26 DREAMS Targets", 2026, "targets", "dreams", "dsnu", + "tNbhYbrKbnk", "FY27 DREAMS Targets", 2027, "targets", "dreams", "dsnu", #Update in November 2024 # For all FY25 SUBNAT/IMPATT, mimic FY24 + "jgp20ElKCMD", "FY27 IMPATT", 2027, "targets", "impatt", "psnu", #Update in November 2024 "jgp20ElKCMD", "FY26 IMPATT", 2026, "targets", "impatt", "psnu", "jgp20ElKCMD", "FY25 IMPATT", 2025, "targets", "impatt", "psnu", "kWKJQYP1uT7", "FY24 IMPATT", 2024, "targets", "impatt", "psnu", + "CMJtVW4ecLn", "FY27 SUBNAT Targets", 2027, "targets", "subnat", "psnu", #Update in November 2024 "CMJtVW4ecLn", "FY26 SUBNAT Targets", 2026, "targets", "subnat", "psnu", "CMJtVW4ecLn", "FY25 SUBNAT Targets", 2025, "targets", "subnat", "psnu", + # Fri Jul 26 15:42:02 2024 -- Touch base with Christian, but believe the belwo is COP23 specific # For all FY23 SUBNAT/IMPATT, remap to FY24 disaggs, as these won't go to # DATIM, but must go to PAW alongside FY24. "kWKJQYP1uT7", "FY23 IMPATT", 2023, "targets", "impatt", "psnu", diff --git a/R/utilities.R b/R/utilities.R index 1c4c04ba..690d7047 100644 --- a/R/utilities.R +++ b/R/utilities.R @@ -310,8 +310,19 @@ getCOPDatasetUids <- function(cop_year, datastreams) { } #List of COP Datasets by year + #Found here https://www.datim.org/dhis-web-maintenance/index.html#/list/dataSetSection/dataSet + #mer_targets NEEDS UPDATED BEFORE GO LIVE of generation cop_datasets <- list( + "2025" = list(# NOT the COP25 versions NEED To update when released + "mer_targets" = c("HUEzkjkij1", # MER Target Setting: PSNU (Facility + # and Community Combined) (TARGETS) updated 6/28/24 + "tNbhYbrKbnk"), # Host Country Targets: DREAMS (USG) updated 4/9/24 + "mer_results" = NA, + "subnat_targets" = "bKSmkDP5YTc", + "subnat_results" = "fZVvcMSA9mZ", + "impatt" = "kWKJQYP1uT7" + ), "2024" = list( "mer_targets" = c("lHUEzkjkij1", # MER Target Setting: PSNU (Facility and Community Combined) (TARGETS) "tNbhYbrKbnk"), # Host Country Targets: DREAMS (USG) @@ -434,6 +445,7 @@ getMapDataPack_DATIM_DEs_COCs <- function(cop_year, datasource = NULL, year = 1) de_coc_map <- switch(as.character(cop_year), "2023" = cop23_map_DataPack_DATIM_DEs_COCs, "2024" = cop24_map_DataPack_DATIM_DEs_COCs, + "2025" = cop25_map_DataPack_DATIM_DEs_COCs, stop("Invalid COP Year")) } @@ -441,6 +453,7 @@ getMapDataPack_DATIM_DEs_COCs <- function(cop_year, datasource = NULL, year = 1) de_coc_map <- switch(as.character(cop_year), "2023" = cop23_map_DataPack_DATIM_DEs_COCs, "2024" = cop24_map_DataPack_DATIM_DEs_COCs, + "2025" = cop25_map_DataPack_DATIM_DEs_COCs, stop("Invalid COP Year")) } diff --git a/R/writeHomeTab.R b/R/writeHomeTab.R index 49948c4a..dd3d8ff0 100644 --- a/R/writeHomeTab.R +++ b/R/writeHomeTab.R @@ -46,7 +46,7 @@ writeHomeTab <- function(wb = NULL, rows = 2, cols = 2) # Title #### - if (cop_year %in% c(2023, 2024) && tool == "Data Pack") { + if (cop_year %in% c(2023, 2024, 2025) && tool == "Data Pack") { tool_title <- "Target Setting Tool" } else { tool_title <- tool @@ -54,7 +54,7 @@ writeHomeTab <- function(wb = NULL, pd <- "COP" yr <- cop_year - 2000 - if (cop_year %in% c(2024)) { + if (cop_year %in% c(2024)) { #Does this need updated to included 2025? pd <- "FY" yr <- yr + 1 } diff --git a/data-raw/COP25/COP25_Data_Pack_generation_script.R b/data-raw/COP25/COP25_Data_Pack_generation_script.R new file mode 100644 index 00000000..fa4392f6 --- /dev/null +++ b/data-raw/COP25/COP25_Data_Pack_generation_script.R @@ -0,0 +1,60 @@ +library(datapackr) + +# Point to DATIM login secrets #### +secrets <- Sys.getenv("SECRETS_FOLDER") %>% paste0(., "datim.json") +datimutils::loginToDATIM(secrets) + +output_folder <- Sys.getenv("OUTPUT_FOLDER") %>% paste0(., "Beta Packs/") +model_data_path <- Sys.getenv("MODEL_DATA_PATH") + +# For Generating Individual Data Packs #### +generation_list <- c("Malawi", + "Zambia", + "Rwanda", + "Mozambique", + "Zimbabwe", + "Tajikistan", + "India", + "Thailand", + "Vietnam", + "South Africa", + "Central America and Brazil", + "Caribbean Region", + "Cameroon", + "Cote d'Ivoire", + "Ghana", + "Benin") + +pick <- datapackr::cop25_datapack_countries %>% + dplyr::filter(datapack_name %in% generation_list) + +# test valid org units against cached #### +valid_OrgUnits <- getDataPackOrgUnits(use_cache = FALSE) + +compare_diffs <- datapackr::valid_OrgUnits_2025 %>% + dplyr::full_join(valid_OrgUnits, by = "uid") %>% + dplyr::filter(is.na(name.x) | is.na(name.y)) + +if (NROW(compare_diffs) > 0) { + stop("Valid org units are not up to date! Please update valid org units.") +} else { + rm(valid_OrgUnits, compare_diffs) +} + +# # For Production run #### +pick <- datapackr::cop25_datapack_countries %>% + dplyr::filter(!datapack_name %in% c("Asia Region", "Western Hemisphere Region", "Turkmenistan")) + +# Execution #### +for (i in seq_along(pick$datapack_name)) { + print(paste0(i, " of ", NROW(pick), ": ", pick[[i, 1]])) + + d <- packTool(model_data_path = model_data_path, + tool = "Data Pack", + datapack_name = pick$datapack_name[i], + country_uids = unlist(pick$country_uids[i]), + template_path = NULL, + cop_year = 2025, + output_folder = output_folder, + results_archive = FALSE) +} diff --git a/data-raw/COP25/COP25_Data_Pack_processing_script.R b/data-raw/COP25/COP25_Data_Pack_processing_script.R new file mode 100644 index 00000000..0ca242f4 --- /dev/null +++ b/data-raw/COP25/COP25_Data_Pack_processing_script.R @@ -0,0 +1,18 @@ +library(datapackr) +library(magrittr) + +# Point to DATIM login secrets #### +secrets <- Sys.getenv("SECRETS_FOLDER") %>% paste0(., "datim.json") + +datimutils::loginToDATIM(secrets) + +output_folder <- Sys.getenv("OUTPUT_FOLDER") %>% paste0(., "COP24 Data Packs/") +model_data_path <- Sys.getenv("MODEL_DATA_PATH") +snuxim_model_data_path <- Sys.getenv("SNUXIM_MODEL_DATA_PATH") + +# Unpack Submitted Data Pack #### +d <- unPackTool(cop_year = 2025, season = "COP") + +d <- checkAnalytics(d, model_data_path) + +d <- writePSNUxIM(d, snuxim_model_data_path, output_folder) diff --git a/data-raw/COP25/update_cop25_de_coc_co_map.R b/data-raw/COP25/update_cop25_de_coc_co_map.R new file mode 100644 index 00000000..c0aed4bc --- /dev/null +++ b/data-raw/COP25/update_cop25_de_coc_co_map.R @@ -0,0 +1,65 @@ +# Point to DATIM login secrets ---- +secrets <- Sys.getenv("SECRETS_FOLDER") %>% paste0(., "datim.json") +datimutils::loginToDATIM(secrets) +cop_year <- 2025 + +# Patch until MER 3.0 deployed +# dp_map <- datapackr::cop23_map_DataPack_DATIM_DEs_COCs %>% +# dplyr::mutate( +# FY = FY + 1, +# period = dplyr::case_when( +# period == "2022Oct" ~ "2023Oct", +# period == "2023Oct" ~ "2024Oct", +# period == "2024Oct" ~ "2025Oct", +# TRUE ~ period), +# period_dataset = stringr::str_replace(period_dataset, "FY25", "FY26"), +# period_dataset = stringr::str_replace(period_dataset, "FY24", "FY25"), +# period_dataset = stringr::str_replace(period_dataset, "FY23", "FY24"), +# ) + +dp_map <- datapackr::update_de_coc_co_map(cop_year, + d2_session = dynGet("d2_default_session", + inherits = TRUE)) + +# Compare old and new maps for accuracy #### +new <- dp_map %>% + dplyr::select(-categoryoption_specified) + +compare_diffs <- datapackr::cop25_map_DataPack_DATIM_DEs_COCs %>% + dplyr::select(-categoryoption_specified) %>% + dplyr::full_join(new, by = c("indicator_code", + "dataelementuid", + "categoryoptioncombouid", + "FY", + "valid_ages.name", "valid_ages.id", "valid_sexes.name", + "valid_sexes.id", "valid_kps.name", "valid_kps.id", + "categoryOptions.ids", "support_type", "resultstatus", "resultstatus_inclusive")) %>% + dplyr::filter(is.na(indicator_code) | is.na(dataelementname.x) | is.na(dataelementname.y)) + +waldo::compare(datapackr::cop25_map_DataPack_DATIM_DEs_COCs, + dp_map, + max_diffs = Inf) + + +cop25_map_DataPack_DATIM_DEs_COCs <- dp_map +usethis::use_data(cop25_map_DataPack_DATIM_DEs_COCs, overwrite = TRUE, compress = "xz") + +## Rebuild package again. (Cmd+Shift+B) + +## Save metadata in API for easy access by Data Management Team + +shareable <- datapackr::cop25_map_DataPack_DATIM_DEs_COCs %>% + dplyr::select(dataElement = dataelementuid, + period, + categoryOptionCombo = categoryoptioncombouid) + +output_folder <- paste0(rprojroot::find_package_root_file(), + "/data-raw/") + +filename <- "cop25_metadata_DEsCOCs" + +filepath <- paste0(output_folder, filename, ".csv") + +utils::write.csv(shareable, filepath, row.names = FALSE) + +## Rebuild package again. (Cmd+Shift+B) diff --git a/data-raw/COP25/update_cop25_psnuxim_schema.R b/data-raw/COP25/update_cop25_psnuxim_schema.R index 96d0de78..ed8c23bf 100644 --- a/data-raw/COP25/update_cop25_psnuxim_schema.R +++ b/data-raw/COP25/update_cop25_psnuxim_schema.R @@ -21,6 +21,6 @@ checkSchema(schema = cop25_psnuxim_schema, cop_year = 2025, tool = "PSNUxIM") -usethis::use_data(cop24_psnuxim_schema, overwrite = TRUE, compress = "xz") +usethis::use_data(cop25_psnuxim_schema, overwrite = TRUE, compress = "xz") ## Rebuild package again. (Cmd+Shift+B) diff --git a/data/cop25_map_DataPack_DATIM_DEs_COCs.rda b/data/cop25_map_DataPack_DATIM_DEs_COCs.rda new file mode 100644 index 00000000..3750dfa5 Binary files /dev/null and b/data/cop25_map_DataPack_DATIM_DEs_COCs.rda differ diff --git a/data/cop25_psnuxim_schema.rda b/data/cop25_psnuxim_schema.rda new file mode 100644 index 00000000..4d467f37 Binary files /dev/null and b/data/cop25_psnuxim_schema.rda differ diff --git a/tests/testthat/sheets/COP25_datapack_model_data_random_MW.rds b/tests/testthat/sheets/COP25_datapack_model_data_random_MW.rds new file mode 100644 index 00000000..8b65b051 Binary files /dev/null and b/tests/testthat/sheets/COP25_datapack_model_data_random_MW.rds differ diff --git a/tests/testthat/sheets/COP25_spectrum_data_random_MW.rds b/tests/testthat/sheets/COP25_spectrum_data_random_MW.rds new file mode 100644 index 00000000..bbcae644 Binary files /dev/null and b/tests/testthat/sheets/COP25_spectrum_data_random_MW.rds differ diff --git a/tests/testthat/test-generateCOP25DataPack.R b/tests/testthat/test-generateCOP25DataPack.R new file mode 100644 index 00000000..d3f20c0b --- /dev/null +++ b/tests/testthat/test-generateCOP25DataPack.R @@ -0,0 +1,166 @@ +context("Create a COP25 Target Setting Tool") + +with_mock_api({ + test_that("We can write an COP25 Target Setting tool", { + + template_path <- getTemplate("COP25_Data_Pack_Template.xlsx") + + expect_true(file.exists(template_path)) + + # For Generating Individual Data Packs #### + generation_list <- c("Malawi") + + pick <- datapackr::COP21_datapacks_countries %>% + dplyr::filter(datapack_name %in% generation_list) %>% + dplyr::arrange(datapack_name) + + output_folder <- paste0("/tmp/", stringi::stri_rand_strings(1, 20)) + dir.create(output_folder) + + #Suppress console output +# Tue Jul 30 13:11:27 2024 - Need to update when time arrives. + spectrum_data <- readRDS(test_sheet("COP25_spectrum_data_random_MW.rds")) + + d <- packTool(model_data_path = test_sheet("COP25_datapack_model_data_random_MW.rds"), + tool = "Data Pack", + datapack_name = pick$datapack_name[1], + country_uids = unlist(pick$country_uids[1]), + template_path = template_path, + cop_year = 2025, + output_folder = output_folder, + results_archive = FALSE, + expand_formulas = TRUE, + spectrum_data = spectrum_data, + d2_session = training) + + expect_setequal(names(d), c("keychain", "info", "tool", "data")) + expect_equal(d$info$datapack_name, "Malawi") + + #Open the generated tool in libreoffice to kick off the formulas + #Do not even try and do this on Windows + skip_if(Sys.info()["sysname"] == "Windows") + + #MacOS users will need to install LibreOffice + lo_path <- ifelse(Sys.info()["sysname"] == "Darwin", + "/Applications/LibreOffice.app/Contents/MacOS/soffice", + #Needs to be relative, but can't figure out terminal command + #got to ls /Applications/ | grep -i libre + system("which libreoffice", intern = TRUE)) + + #Skip this if we cannot execute libreoffice + skip_if(file.access(lo_path, 1) != 0) + + out_dir <- paste0(output_folder, "/out") + dir.create(out_dir) + + Sys.setenv(LD_LIBRARY_PATH = ifelse(Sys.info()["sysname"] == "Darwin", + "/Applications/LibreOffice.app/Contents/MacOS/soffice", + "/usr/lib/libreoffice/program/")) + sys_command <- paste0(ifelse(Sys.info()["sysname"] == "Darwin", + "/Applications/LibreOffice.app/Contents/MacOS/soffice", + "libreoffice"), + " --headless --convert-to xlsx --outdir ", out_dir, " '", d$info$output_file, "'") + system(sys_command) + + out_file <- paste0(out_dir, "/", basename(d$info$output_file)) + + #Unpack this tool which has been "opened" in libreoffice + d_opened <- unPackTool(submission_path = out_file, d2_session = training) + + expect_identical(d$info$datapack_name, d_opened$info$datapack_name) + expect_setequal(names(d_opened), c("keychain", "info", "data", "tests", "datim", "sheets")) + expect_true(NROW(d_opened$data$analytics) > 0) + expect_true(all(d_opened$data$analytics$mechanism_desc == "default")) + + + d_data_targets_names <- c("PSNU", "psnuid", "sheet_name", "indicator_code", "Age", "Sex", "KeyPop", "value") + d_data_tests_types <- c("tbl_df", "tbl", "data.frame") + + + d <- unPackSheets(d_opened, check_sheets = TRUE) + + + expect_true(!is.null(d_opened$data$MER)) + expect_setequal(class(d_opened$data$MER), c("tbl_df", "tbl", "data.frame")) + expect_identical(unname(sapply(d_opened$data$MER, typeof)), c(rep("character", 7), "double")) + expect_setequal(names(d_opened$data$MER), d_data_targets_names) + expect_true((NROW(d_opened$data$MER) > 0)) + + expect_true(!is.null(d_opened$data$SUBNAT_IMPATT)) + expect_setequal(class(d_opened$data$SUBNAT_IMPATT), c("tbl_df", "tbl", "data.frame")) + expect_identical(unname(sapply(d_opened$data$SUBNAT_IMPATT, typeof)), c(rep("character", 7), "double")) + expect_setequal(names(d_opened$data$SUBNAT_IMPATT), d_data_targets_names) + expect_true((NROW(d_opened$data$SUBNAT_IMPATT) > 0)) + + + validation_summary <- validationSummary(d_opened) + expect_named(validation_summary, + c("count", "country_name", "country_uid", + "ou", "ou_id", "test_name", "validation_issue_category"), + ignore.order = TRUE) + + + #DP-837 + #Specific test of AGYW_PREV orgunits + agyw_have <- d$sheets$AGYW %>% + dplyr::select(PSNU) %>% + dplyr::distinct() %>% + dplyr::mutate(psnu_uid = stringr::str_extract(PSNU, "(?<=(\\(|\\[))([A-Za-z][A-Za-z0-9]{10})(?=(\\)|\\])$)")) %>% + dplyr::arrange(PSNU) + + agyw_want <- getValidOrgUnits("2025") %>% + dplyr::filter(country_uid %in% d$info$country_uids) %>% + add_dp_label(., "2025") %>% + dplyr::arrange(dp_label) %>% + dplyr::filter(!is.na(DREAMS)) %>% + dplyr::select(PSNU = dp_label, psnu_uid = uid) %>% + dplyr::arrange(PSNU) + + expect_identical(agyw_want, agyw_have) + + + #Check the PSNUs in normal sheets, excluding the PSNUxIM tab, Year 2 and AGYW + discard_names <- function(l, kn) { + l[!(names(l) %in% kn)] + } + + extract_PSNU <- function(df) { + df %>% + dplyr::select(PSNU) %>% + dplyr::distinct() %>% + dplyr::mutate(psnu_uid = + stringr::str_extract(PSNU, "(?<=(\\(|\\[))([A-Za-z][A-Za-z0-9]{10})(?=(\\)|\\])$)")) %>% + dplyr::arrange(PSNU) + } + + sheet_psnus <- d$sheets %>% + discard_names(c("PSNUxIM", "Year 2", "AGYW")) %>% + purrr::map(extract_PSNU) + + wanted_psnus <- + getValidOrgUnits("2025") %>% + dplyr::filter(country_uid %in% d$info$country_uids) %>% + add_dp_label(., "2025") %>% + dplyr::arrange(dp_label) %>% + ## Remove DSNUs + dplyr::filter(org_type != "DSNU") %>% + dplyr::select(PSNU = dp_label, psnu_uid = uid) + + expect_true(all(unlist(purrr::map(sheet_psnus, identical, wanted_psnus)))) + + #DP-970--Duplicates in the Year2 tab + duplicated_export_rows <- d$datim$year2 %>% + dplyr::select(dataElement, period, orgUnit, categoryOptionCombo, attributeOptionCombo) %>% + dplyr::group_by_all() %>% + dplyr::mutate(n = dplyr::n()) %>% + dplyr::filter(n > 1) %>% + NROW() + + expect_equal(duplicated_export_rows, 0L) + + + #There should be no zeros in d$data$SNUxIM except for dedpe + expect_false(any(d$data$SNUxIM[d$data$SNUxIM$value == 0 & !grepl("^0000[01]", d$data$SNUxIM$mech_code), ])) + + }) +})