From bfde97690986062c1a5cc43aae6fe3c123bc51a7 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Tue, 21 Sep 2021 13:35:56 +0200 Subject: [PATCH] Migrate web monitors to Upptime --- .../sdmxdl/connectors/drivers/AbsDriver.java | 2 +- .../connectors/drivers/EurostatDriver.java | 2 +- .../sdmxdl/connectors/drivers/ImfDriver.java | 2 +- .../connectors/drivers/InseeDriver.java | 2 +- .../sdmxdl/connectors/drivers/NbbDriver.java | 2 +- .../sdmxdl/connectors/drivers/OecdDriver.java | 2 +- .../connectors/drivers/Sdmx21Driver.java | 28 ++-- .../sdmxdl/connectors/drivers/SeDriver.java | 2 +- .../sdmxdl/connectors/drivers/UisDriver.java | 2 +- sdmx-dl-provider-ri/pom.xml | 5 + .../sdmxdl/ri/web/drivers/AbsDriver2.java | 2 +- .../sdmxdl/ri/web/drivers/BbkDriver.java | 2 +- .../sdmxdl/ri/web/drivers/DotStatDriver2.java | 8 +- .../sdmxdl/ri/web/drivers/NbbDriver2.java | 2 +- .../sdmxdl/ri/web/drivers/Sdmx21Driver2.java | 28 ++-- .../sdmxdl/ri/web/monitors/UpptimeId.java | 46 +++++++ .../ri/web/monitors/UpptimeMonitoring.java | 126 ++++++++++++++++++ .../src/main/java/module-info.java | 6 + .../sdmxdl/ri/web/monitors/UpptimeIdTest.java | 48 +++++++ .../web/monitors/UpptimeMonitoringTest.java | 37 +++++ .../sdmxdl/ri/web/monitors/summary.json | 41 ++++++ 21 files changed, 352 insertions(+), 43 deletions(-) create mode 100644 sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeId.java create mode 100644 sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoring.java create mode 100644 sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeIdTest.java create mode 100644 sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoringTest.java create mode 100644 sdmx-dl-provider-ri/src/test/resources/internal/sdmxdl/ri/web/monitors/summary.json diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/AbsDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/AbsDriver.java index 080be7c2e..64a4b1c90 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/AbsDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/AbsDriver.java @@ -45,7 +45,7 @@ public final class AbsDriver implements SdmxWebDriver { .driver(CONNECTORS_ABS) .endpointOf("http://stat.data.abs.gov.au/restsdmx/sdmx.ashx") .websiteOf("http://stat.data.abs.gov.au") - .monitorOf("UptimeRobot", "m783847060-975767bc3a033ea3f3ac8ca2") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ABS") .build()) .build(); } diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/EurostatDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/EurostatDriver.java index f7f4cdc45..e81de0390 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/EurostatDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/EurostatDriver.java @@ -46,7 +46,7 @@ public final class EurostatDriver implements SdmxWebDriver { .driver(CONNECTORS_EUROSTAT) .endpointOf("https://ec.europa.eu/eurostat/SDMX/diss-web/rest") .websiteOf("https://ec.europa.eu/eurostat/data/database") - .monitorOf("UptimeRobot", "m783847077-390f706bd3acf8fb640e48df") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ESTAT") .build()) .build(); } diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/ImfDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/ImfDriver.java index 3166979e8..695b73315 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/ImfDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/ImfDriver.java @@ -45,7 +45,7 @@ public final class ImfDriver implements SdmxWebDriver { .driver(CONNECTORS_IMF) .endpointOf("http://dataservices.imf.org/REST/SDMX_XML.svc") .websiteOf("https://data.imf.org") - .monitorOf("UptimeRobot", "m783847087-0ea5fc62a2cf830952d3dd98") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:IMF") .build()) .build(); } diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/InseeDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/InseeDriver.java index b4e3d4cf5..c8143667e 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/InseeDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/InseeDriver.java @@ -59,7 +59,7 @@ public final class InseeDriver implements SdmxWebDriver { .driver(CONNECTORS_INSEE) .endpointOf("https://bdm.insee.fr/series/sdmx") .websiteOf("https://www.insee.fr/fr/statistiques") - .monitorOf("UptimeRobot", "m783847118-b9c18640231d06a23fd6e386") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:INSEE") .build()) .build(); diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/NbbDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/NbbDriver.java index b2d2ff89f..fcdb8a9fd 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/NbbDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/NbbDriver.java @@ -45,7 +45,7 @@ public final class NbbDriver implements SdmxWebDriver { .driver(CONNECTORS_NBB) .endpointOf("https://stat.nbb.be/restsdmx/sdmx.ashx") .websiteOf("https://stat.nbb.be") - .monitorOf("UptimeRobot", "m783847132-29c6aa5b9cf34b10a6466571") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:NBB") .build()) .build(); } diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/OecdDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/OecdDriver.java index 25aeb8060..dbe4b782f 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/OecdDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/OecdDriver.java @@ -45,7 +45,7 @@ public final class OecdDriver implements SdmxWebDriver { .driver(CONNECTORS_OECD) .endpointOf("https://stats.oecd.org/restsdmx/sdmx.ashx") .websiteOf("https://stats.oecd.org") - .monitorOf("UptimeRobot", "m783847142-c1c97c88fd0d958b6478d961") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:OECD") .build()) .build(); } diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/Sdmx21Driver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/Sdmx21Driver.java index df06e6fd4..16e7c1d8d 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/Sdmx21Driver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/Sdmx21Driver.java @@ -57,7 +57,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .endpointOf("https://stats.bis.org/api/v1") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://stats.bis.org/statx/toc/LBS.html") - .monitorOf("UptimeRobot", "m787932103-5b1ea6eac87a4f436c565169") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:BIS") .build()) .source(SdmxWebSource .builder() @@ -66,7 +66,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .driver(CONNECTORS_SDMX_21) .endpointOf("https://nsiws-stable-camstat-live.officialstatistics.org/rest") .websiteOf("http://camstat.nis.gov.kh/?locale=en&start=0") - .monitorOf("UptimeRobot", "m788487928-37e8ae2f2d754f577b97a780") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:CAMSTAT") .build()) .source(SdmxWebSource .builder() @@ -78,7 +78,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .propertyOf(SUPPORTS_COMPRESSION_PROPERTY, true) .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://sdw.ecb.europa.eu") - .monitorOf("UptimeRobot", "m783846981-b55d7e635c5cdc16e16bac2a") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ECB") .build()) .source(SdmxWebSource .builder() @@ -87,7 +87,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .driver(CONNECTORS_SDMX_21) .endpointOf("https://api-dataexplorer.unescap.org/rest/") .websiteOf("https://dataexplorer.unescap.org/") - .monitorOf("UptimeRobot", "m788486016-99b7fd6f51508b689c67d460") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ESCAP") .build()) .source(SdmxWebSource .builder() @@ -97,7 +97,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .endpointOf("https://www.ilo.org/sdmx/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://ilostat.ilo.org/data/") - .monitorOf("UptimeRobot", "m783847083-609d3e4ebc1da9455baeb63e") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ILO") .build()) .source(SdmxWebSource .builder() @@ -108,7 +108,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .propertyOf(SUPPORTS_COMPRESSION_PROPERTY, true) .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://data.imf.org") - .monitorOf("UptimeRobot", "m783847101-ba94f8b8442fc1c13a36ad89") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:IMF_SDMX_CENTRAL") .build()) .source(SdmxWebSource .builder() @@ -117,7 +117,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .driver(CONNECTORS_SDMX_21) .endpointOf("https://sdmx.snieg.mx/service/Rest") .websiteOf("https://sdmx.snieg.mx") - .monitorOf("UptimeRobot", "m783847104-047f5d4a8e6dacc0effb488b") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:INEGI") .build()) .source(SdmxWebSource .builder() @@ -128,7 +128,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .propertyOf(SUPPORTS_COMPRESSION_PROPERTY, true) .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://www.istat.it/en/analysis-and-products") - .monitorOf("UptimeRobot", "m783847124-82c4c955d73e33fa148f72b8") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ISTAT") .build()) .source(SdmxWebSource .builder() @@ -138,7 +138,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .endpointOf("https://data.norges-bank.no/api") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://www.norges-bank.no/en/topics/Statistics/") - .monitorOf("UptimeRobot", "m787932098-8e50275369b8d0e7bdc64354") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:NB") .build()) .source(SdmxWebSource .builder() @@ -148,7 +148,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .endpointOf("https://registry.sdmx.org/ws/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://registry.sdmx.org/overview.html") - .monitorOf("UptimeRobot", "m788488202-23b752a1d418181c2d64d6d1") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:SGR") .build()) .source(SdmxWebSource .builder() @@ -158,7 +158,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .endpointOf("https://stats-nsi-stable.pacificdata.org/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://stats.pacificdata.org/?locale=en") - .monitorOf("UptimeRobot", "m788488402-48b2f77f10060fcc6bb4baaa") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:SPC") .build()) .source(SdmxWebSource .builder() @@ -168,7 +168,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .endpointOf("https://data.un.org/WS/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://data.un.org/SdmxBrowser/start") - .monitorOf("UptimeRobot", "m783847155-179810527371702e70b1c1b3") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:UNDATA") .build()) .source(SdmxWebSource .builder() @@ -178,7 +178,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .endpointOf("https://api.worldbank.org/v2/sdmx/rest") .propertyOf(SUPPORTS_COMPRESSION_PROPERTY, true) .websiteOf("https://data.worldbank.org") - .monitorOf("UptimeRobot", "m783847161-28762547004598b9cc9311bc") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:WB") .build()) .source(SdmxWebSource .builder() @@ -187,7 +187,7 @@ public final class Sdmx21Driver implements SdmxWebDriver { .driver(CONNECTORS_SDMX_21) .endpointOf("http://wits.worldbank.org/API/V1/SDMX/V21/rest") .websiteOf("https://wits.worldbank.org") - .monitorOf("UptimeRobot", "m783847171-e363f4fe1930586228b1dc39") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:WITS") .build()) .build(); diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/SeDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/SeDriver.java index 3ba1ab9cf..3015fb765 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/SeDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/SeDriver.java @@ -48,7 +48,7 @@ public final class SeDriver implements SdmxWebDriver { .driver(CONNECTORS_ES) .endpointOf("http://andmebaas.stat.ee/restsdmx/sdmx.ashx") .websiteOf("http://andmebaas.stat.ee") - .monitorOf("UptimeRobot", "m783847065-e1d117d7c62688abdfec4734") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:SE") .build()) .build(); diff --git a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/UisDriver.java b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/UisDriver.java index a7a97a652..f1c4815d2 100644 --- a/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/UisDriver.java +++ b/sdmx-dl-provider-connectors/src/main/java/internal/sdmxdl/connectors/drivers/UisDriver.java @@ -51,7 +51,7 @@ public final class UisDriver implements SdmxWebDriver { .driver(CONNECTORS_UIS) .endpointOf(FALLBACK_ENDPOINT) .websiteOf("http://data.uis.unesco.org") - .monitorOf("UptimeRobot", "m783847149-7753fbe93eefc48f2ac9983f") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:UIS") .build()) .build(); diff --git a/sdmx-dl-provider-ri/pom.xml b/sdmx-dl-provider-ri/pom.xml index 2e733add6..b914e3427 100644 --- a/sdmx-dl-provider-ri/pom.xml +++ b/sdmx-dl-provider-ri/pom.xml @@ -65,6 +65,11 @@ picocsv 2.0.0 + + com.jsoniter + jsoniter + 0.9.23 + diff --git a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/AbsDriver2.java b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/AbsDriver2.java index 29d82e5e7..919891edc 100644 --- a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/AbsDriver2.java +++ b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/AbsDriver2.java @@ -61,7 +61,7 @@ public final class AbsDriver2 implements SdmxWebDriver { .driver(RI_ABS) .endpointOf("https://stat.data.abs.gov.au/restsdmx/sdmx.ashx") .websiteOf("https://stat.data.abs.gov.au") - .monitorOf("UptimeRobot", "m783847060-975767bc3a033ea3f3ac8ca2") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ABS") .build()) .build(); diff --git a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/BbkDriver.java b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/BbkDriver.java index b16a586e1..a48813469 100644 --- a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/BbkDriver.java +++ b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/BbkDriver.java @@ -68,7 +68,7 @@ public final class BbkDriver implements SdmxWebDriver { .driver(RI_BBK) .endpointOf("https://api.statistiken.bundesbank.de/rest") .websiteOf("https://www.bundesbank.de/en/statistics/time-series-databases") - .monitorOf("UptimeRobot", "m788346159-bf4da8fbc0d9c633266a9387") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:BBK") .build()) .build(); diff --git a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/DotStatDriver2.java b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/DotStatDriver2.java index 46a0a3e48..06ebc0411 100644 --- a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/DotStatDriver2.java +++ b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/DotStatDriver2.java @@ -55,7 +55,7 @@ public final class DotStatDriver2 implements SdmxWebDriver { .driver(RI_DOTSTAT) .endpointOf("https://stats.oecd.org/restsdmx/sdmx.ashx") .websiteOf("https://stats.oecd.org") - .monitorOf("UptimeRobot", "m783847142-c1c97c88fd0d958b6478d961") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:OECD") .build()) .source(SdmxWebSource .builder() @@ -64,7 +64,7 @@ public final class DotStatDriver2 implements SdmxWebDriver { .driver(RI_DOTSTAT) .endpointOf("http://andmebaas.stat.ee/restsdmx/sdmx.ashx") .websiteOf("http://andmebaas.stat.ee") - .monitorOf("UptimeRobot", "m783847065-e1d117d7c62688abdfec4734") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:SE") .build()) .source(SdmxWebSource .builder() @@ -73,7 +73,7 @@ public final class DotStatDriver2 implements SdmxWebDriver { .driver(RI_DOTSTAT) .endpointOf(UIS_ENDPOINT) .websiteOf("http://data.uis.unesco.org") - .monitorOf("UptimeRobot", "m783847149-7753fbe93eefc48f2ac9983f") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:UIS") .build()) .source(SdmxWebSource .builder() @@ -82,7 +82,7 @@ public final class DotStatDriver2 implements SdmxWebDriver { .driver(RI_DOTSTAT) .endpointOf("https://stats2.digitalresources.jisc.ac.uk/restsdmx/sdmx.ashx") .websiteOf("https://stats2.digitalresources.jisc.ac.uk/") - .monitorOf("UptimeRobot", "m788496292-8f7f10200ba0cf792589e612") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:UKDS") .build()) .build(); diff --git a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/NbbDriver2.java b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/NbbDriver2.java index 497a59b7d..cfe858407 100644 --- a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/NbbDriver2.java +++ b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/NbbDriver2.java @@ -72,7 +72,7 @@ public final class NbbDriver2 implements SdmxWebDriver { .driver(RI_NBB) .endpointOf("https://stat.nbb.be/restsdmx/sdmx.ashx") .websiteOf("https://stat.nbb.be") - .monitorOf("UptimeRobot", "m783847132-29c6aa5b9cf34b10a6466571") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:NBB") .build()) .build(); diff --git a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/Sdmx21Driver2.java b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/Sdmx21Driver2.java index 8cf2a35b9..ba755bcff 100644 --- a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/Sdmx21Driver2.java +++ b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/drivers/Sdmx21Driver2.java @@ -62,7 +62,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://stats.bis.org/api/v1") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://stats.bis.org/statx/toc/LBS.html") - .monitorOf("UptimeRobot", "m787932103-5b1ea6eac87a4f436c565169") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:BIS") .build()) .source(SdmxWebSource .builder() @@ -71,7 +71,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .driver(RI_SDMX_21) .endpointOf("https://nsiws-stable-camstat-live.officialstatistics.org/rest") .websiteOf("http://camstat.nis.gov.kh/?locale=en&start=0") - .monitorOf("UptimeRobot", "m788487928-37e8ae2f2d754f577b97a780") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:CAMSTAT") .build()) .source(SdmxWebSource .builder() @@ -82,7 +82,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://sdw-wsrest.ecb.europa.eu/service") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://sdw.ecb.europa.eu") - .monitorOf("UptimeRobot", "m783846981-b55d7e635c5cdc16e16bac2a") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ECB") .build()) .source(SdmxWebSource .builder() @@ -91,7 +91,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .driver(RI_SDMX_21) .endpointOf("https://api-dataexplorer.unescap.org/rest/") .websiteOf("https://dataexplorer.unescap.org/") - .monitorOf("UptimeRobot", "m788486016-99b7fd6f51508b689c67d460") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ESCAP") .build()) .source(SdmxWebSource .builder() @@ -101,7 +101,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://www.ilo.org/sdmx/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://ilostat.ilo.org/data/") - .monitorOf("UptimeRobot", "m783847083-609d3e4ebc1da9455baeb63e") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ILO") .build()) .source(SdmxWebSource .builder() @@ -111,7 +111,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://sdmxcentral.imf.org/ws/public/sdmxapi/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://data.imf.org") - .monitorOf("UptimeRobot", "m783847101-ba94f8b8442fc1c13a36ad89") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:IMF_SDMX_CENTRAL") .build()) .source(SdmxWebSource .builder() @@ -122,7 +122,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .propertyOf(TRAILING_SLASH_REQUIRED_PROPERTY, true) .websiteOf("https://sdmx.snieg.mx") - .monitorOf("UptimeRobot", "m783847104-047f5d4a8e6dacc0effb488b") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:INEGI") .build()) .source(SdmxWebSource .builder() @@ -132,7 +132,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("http://sdmx.istat.it/SDMXWS/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://www.istat.it/en/analysis-and-products") - .monitorOf("UptimeRobot", "m783847124-82c4c955d73e33fa148f72b8") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:ISTAT") .build()) .source(SdmxWebSource .builder() @@ -142,7 +142,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://data.norges-bank.no/api") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://www.norges-bank.no/en/topics/Statistics/") - .monitorOf("UptimeRobot", "m787932098-8e50275369b8d0e7bdc64354") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:NB") .build()) .source(SdmxWebSource .builder() @@ -152,7 +152,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://registry.sdmx.org/ws/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://registry.sdmx.org/overview.html") - .monitorOf("UptimeRobot", "m788488202-23b752a1d418181c2d64d6d1") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:SGR") .build()) .source(SdmxWebSource .builder() @@ -162,7 +162,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://stats-nsi-stable.pacificdata.org/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://stats.pacificdata.org/?locale=en") - .monitorOf("UptimeRobot", "m788488402-48b2f77f10060fcc6bb4baaa") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:SPC") .build()) .source(SdmxWebSource .builder() @@ -172,7 +172,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("https://data.un.org/WS/rest") .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .websiteOf("https://data.un.org/SdmxBrowser/start") - .monitorOf("UptimeRobot", "m783847155-179810527371702e70b1c1b3") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:UNDATA") .build()) .source(SdmxWebSource .builder() @@ -183,7 +183,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .propertyOf(DETAIL_SUPPORTED_PROPERTY, true) .propertyOf(TRAILING_SLASH_REQUIRED_PROPERTY, true) .websiteOf("https://data.worldbank.org") - .monitorOf("UptimeRobot", "m783847161-28762547004598b9cc9311bc") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:WB") .build()) .source(SdmxWebSource .builder() @@ -193,7 +193,7 @@ public final class Sdmx21Driver2 implements SdmxWebDriver { .endpointOf("http://wits.worldbank.org/API/V1/SDMX/V21/rest") .propertyOf(TRAILING_SLASH_REQUIRED_PROPERTY, true) .websiteOf("https://wits.worldbank.org") - .monitorOf("UptimeRobot", "m783847171-e363f4fe1930586228b1dc39") + .monitorOf("Upptime", "nbbrd:sdmx-upptime:WITS") .build()) .build(); diff --git a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeId.java b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeId.java new file mode 100644 index 000000000..d46fca506 --- /dev/null +++ b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeId.java @@ -0,0 +1,46 @@ +package internal.sdmxdl.ri.web.monitors; + +import internal.util.rest.RestQueryBuilder; +import nbbrd.design.StringValue; +import org.checkerframework.checker.nullness.qual.NonNull; + +import java.net.MalformedURLException; +import java.net.URL; + +@StringValue +@lombok.Value +public class UpptimeId { + + @lombok.NonNull + String owner; + + @lombok.NonNull + String repo; + + @lombok.NonNull + String site; + + @Override + public String toString() { + return owner + ":" + repo + ":" + site; + } + + public @NonNull URL toSummaryURL() throws MalformedURLException { + return RestQueryBuilder + .of(new URL("https://raw.githubusercontent.com")) + .path(owner) + .path(repo) + .path("master") + .path("history") + .path("summary.json") + .build(); + } + + public static @NonNull UpptimeId parse(@NonNull CharSequence id) throws IllegalArgumentException { + String[] items = id.toString().split(":", -1); + if (items.length != 3) { + throw new IllegalArgumentException("Cannot parse id; expected 3 parts, found " + items.length); + } + return new UpptimeId(items[0], items[1], items[2]); + } +} diff --git a/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoring.java b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoring.java new file mode 100644 index 000000000..cb3f5a832 --- /dev/null +++ b/sdmx-dl-provider-ri/src/main/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoring.java @@ -0,0 +1,126 @@ +package internal.sdmxdl.ri.web.monitors; + +import com.jsoniter.JsonIterator; +import internal.sdmxdl.ri.web.RestClients; +import internal.util.rest.HttpRest; +import internal.util.rest.MediaType; +import nbbrd.design.MightBePromoted; +import nbbrd.design.VisibleForTesting; +import nbbrd.io.text.Parser; +import nbbrd.service.ServiceProvider; +import org.checkerframework.checker.nullness.qual.NonNull; +import sdmxdl.LanguagePriorityList; +import sdmxdl.web.SdmxWebMonitor; +import sdmxdl.web.SdmxWebMonitorReport; +import sdmxdl.web.SdmxWebSource; +import sdmxdl.web.SdmxWebStatus; +import sdmxdl.web.spi.SdmxWebContext; +import sdmxdl.web.spi.SdmxWebMonitoring; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.NumberFormat; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +@ServiceProvider +public class UpptimeMonitoring implements SdmxWebMonitoring { + + @Override + public @NonNull String getProviderName() { + return "Upptime"; + } + + @Override + public @NonNull SdmxWebMonitorReport getReport(@NonNull SdmxWebSource source, @NonNull SdmxWebContext context) throws IOException, IllegalArgumentException { + checkMonitor(source.getMonitor()); + + UpptimeId id = UpptimeId.parse(source.getMonitor().getId()); + + HttpRest.Context restContext = RestClients.getRestContext(source, context); + + HttpRest.Client client = HttpRest.newClient(restContext); + + return getSummary(client, id) + .stream() + .filter(item -> item.getName().equals(id.getSite())) + .map(UpptimeMonitoring::getReport) + .findFirst() + .orElseThrow(IOException::new); + } + + private void checkMonitor(SdmxWebMonitor monitor) { + if (monitor == null) { + throw new IllegalArgumentException("Expecting monitor not to be null"); + } + if (!monitor.getProvider().equals(getProviderName())) { + throw new IllegalArgumentException(monitor.toString()); + } + } + + private static List getSummary(HttpRest.Client client, UpptimeId id) throws IOException { + try (HttpRest.Response response = client.requestGET(id.toSummaryURL(), Collections.singletonList(MediaType.ANY_TYPE), LanguagePriorityList.ANY.toString())) { + try (InputStream body = response.getBody()) { + return SiteSummary.parseAll(toBytes(body)); + } + } + } + + @MightBePromoted + @VisibleForTesting + static byte[] toBytes(InputStream stream) throws IOException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int length; + while ((length = stream.read(buf)) > 0) { + result.write(buf, 0, length); + } + return result.toByteArray(); + } + + @VisibleForTesting + @lombok.Data + static class SiteSummary { + + String name; + String status; + String uptime; + + public static SiteSummary of(String name, String status, String uptime) { + SiteSummary result = new SiteSummary(); + result.setName(name); + result.setStatus(status); + result.setUptime(uptime); + return result; + } + + public static List parseAll(byte[] bytes) { + return Arrays.asList(JsonIterator.deserialize(bytes, SiteSummary[].class)); + } + } + + private static SdmxWebStatus parseStatus(String status) { + switch (status) { + case "up": + return SdmxWebStatus.UP; + case "down": + return SdmxWebStatus.DOWN; + default: + return SdmxWebStatus.UNKNOWN; + } + } + + @VisibleForTesting + static SdmxWebMonitorReport getReport(SiteSummary summary) { + return SdmxWebMonitorReport + .builder() + .status(parseStatus(summary.getStatus())) + .uptimeRatio(NUMBER_PARSER.parseValue(summary.getUptime()).orElse(0).doubleValue()) + .build(); + } + + private static final @NonNull Parser NUMBER_PARSER = Parser.onNumberFormat(NumberFormat.getPercentInstance(Locale.ROOT)); +} diff --git a/sdmx-dl-provider-ri/src/main/java/module-info.java b/sdmx-dl-provider-ri/src/main/java/module-info.java index b6cd8a65b..2170f894a 100644 --- a/sdmx-dl-provider-ri/src/main/java/module-info.java +++ b/sdmx-dl-provider-ri/src/main/java/module-info.java @@ -9,6 +9,7 @@ requires sdmxdl.util.xml; requires com.github.tuupertunut.powershelllibjava; requires nbbrd.picocsv; + requires jsoniter; requires transitive sdmxdl.api; @@ -30,4 +31,9 @@ internal.util.http.curl.CurlHttpURLConnectionFactory; uses internal.util.http.HttpURLConnectionFactory; + + provides sdmxdl.web.spi.SdmxWebMonitoring with + internal.sdmxdl.ri.web.monitors.UpptimeMonitoring; + + opens internal.sdmxdl.ri.web.monitors to jsoniter; } \ No newline at end of file diff --git a/sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeIdTest.java b/sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeIdTest.java new file mode 100644 index 000000000..648f8fe5f --- /dev/null +++ b/sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeIdTest.java @@ -0,0 +1,48 @@ +package internal.sdmxdl.ri.web.monitors; + +import org.junit.Test; + +import java.net.MalformedURLException; +import java.net.URL; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +public class UpptimeIdTest { + + @Test + public void testToString() { + assertThat(new UpptimeId("", "", "").toString()) + .isEqualTo("::"); + + assertThat(new UpptimeId("nbbrd", "sdmx-upptime", "ECB").toString()) + .isEqualTo("nbbrd:sdmx-upptime:ECB"); + } + + @Test + public void toSummaryURL() throws MalformedURLException { + assertThat(new UpptimeId("", "", "").toSummaryURL()) + .isEqualTo(new URL("https://raw.githubusercontent.com///master/history/summary.json")); + + assertThat(new UpptimeId("nbbrd", "sdmx-upptime", "ECB").toSummaryURL()) + .isEqualTo(new URL("https://raw.githubusercontent.com/nbbrd/sdmx-upptime/master/history/summary.json")); + } + + @Test + public void parse() { + assertThat(UpptimeId.parse("::")) + .isEqualTo(new UpptimeId("", "", "")); + + assertThat(UpptimeId.parse("nbbrd:sdmx-upptime:ECB")) + .isEqualTo(new UpptimeId("nbbrd", "sdmx-upptime", "ECB")); + + assertThatIllegalArgumentException() + .isThrownBy(() -> UpptimeId.parse("")); + + assertThatIllegalArgumentException() + .isThrownBy(() -> UpptimeId.parse(":")); + + assertThatIllegalArgumentException() + .isThrownBy(() -> UpptimeId.parse(":::")); + } +} \ No newline at end of file diff --git a/sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoringTest.java b/sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoringTest.java new file mode 100644 index 000000000..0c337ed6d --- /dev/null +++ b/sdmx-dl-provider-ri/src/test/java/internal/sdmxdl/ri/web/monitors/UpptimeMonitoringTest.java @@ -0,0 +1,37 @@ +package internal.sdmxdl.ri.web.monitors; + +import org.junit.Test; +import sdmxdl.web.SdmxWebMonitorReport; +import sdmxdl.web.SdmxWebStatus; + +import java.io.IOException; +import java.io.InputStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.atIndex; + +public class UpptimeMonitoringTest { + + static byte[] loadSample() throws IOException { + try (InputStream stream = UpptimeMonitoringTest.class.getResourceAsStream("summary.json")) { + return UpptimeMonitoring.toBytes(stream); + } + } + + @Test + public void testSiteSummary() throws IOException { + assertThat(UpptimeMonitoring.SiteSummary.parseAll(loadSample())) + .contains(UpptimeMonitoring.SiteSummary.of("ABS", "up", "100.00%"), atIndex(0)) + .contains(UpptimeMonitoring.SiteSummary.of("ILO", "down", "20.97%"), atIndex(1)) + .hasSize(2); + } + + @Test + public void testGetReport() { + assertThat(UpptimeMonitoring.getReport(UpptimeMonitoring.SiteSummary.of("ABS", "up", "100.00%"))) + .isEqualTo(SdmxWebMonitorReport.builder().status(SdmxWebStatus.UP).uptimeRatio(1d).build()); + + assertThat(UpptimeMonitoring.getReport(UpptimeMonitoring.SiteSummary.of("ILO", "down", "20.97%"))) + .isEqualTo(SdmxWebMonitorReport.builder().status(SdmxWebStatus.DOWN).uptimeRatio(.2097d).build()); + } +} diff --git a/sdmx-dl-provider-ri/src/test/resources/internal/sdmxdl/ri/web/monitors/summary.json b/sdmx-dl-provider-ri/src/test/resources/internal/sdmxdl/ri/web/monitors/summary.json new file mode 100644 index 000000000..25735ad23 --- /dev/null +++ b/sdmx-dl-provider-ri/src/test/resources/internal/sdmxdl/ri/web/monitors/summary.json @@ -0,0 +1,41 @@ +[ + { + "name": "ABS", + "url": "https://stat.data.abs.gov.au/restsdmx/sdmx.ashx/GetDataStructure/ALL", + "icon": "https://favicons.githubusercontent.com/stat.data.abs.gov.au", + "slug": "abs", + "status": "up", + "uptime": "100.00%", + "uptimeDay": "100.00%", + "uptimeWeek": "100.00%", + "uptimeMonth": "100.00%", + "uptimeYear": "100.00%", + "time": 4674, + "timeDay": 4674, + "timeWeek": 4674, + "timeMonth": 4674, + "timeYear": 4674, + "dailyMinutesDown": {} + }, + { + "name": "ILO", + "url": "https://www.ilo.org/sdmx/rest/dataflow/all/all/latest", + "icon": "https://favicons.githubusercontent.com/www.ilo.org", + "slug": "ilo", + "status": "down", + "uptime": "20.97%", + "uptimeDay": "20.97%", + "uptimeWeek": "20.97%", + "uptimeMonth": "20.97%", + "uptimeYear": "20.97%", + "time": 14989, + "timeDay": 14989, + "timeWeek": 14989, + "timeMonth": 14989, + "timeYear": 14989, + "dailyMinutesDown": { + "2021-09-21": 412, + "2021-09-20": 491 + } + } +] \ No newline at end of file