Merge branch 'develop'

nbbrd · Oct 16, 2023 · f4e4240 · f4e4240
2 parents 4be28f9 + 2895c0f
commit f4e4240
Show file tree

Hide file tree

Showing 468 changed files with 7,959 additions and 8,745 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -4,8 +4,8 @@ updates:
     target-branch: "develop"
     directory: "/"
     schedule:
-      interval: daily
-      time: "04:00"
+      interval: "weekly"
+      day: "saturday"
     open-pull-requests-limit: 10
   - package-ecosystem: "github-actions"
     target-branch: "develop"

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -35,7 +35,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 0
@@ -56,7 +56,7 @@ jobs:
 
     steps:
       - name: Checkout pages
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: 'gh-pages'
 

diff --git a/.github/workflows/hugo-geekdoc.yml b/.github/workflows/hugo-geekdoc.yml
@@ -0,0 +1,59 @@
+name: Update hugo-geekdoc
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * SAT'
+
+jobs:
+  update-mvn-property:
+    runs-on: ubuntu-latest
+    name: Update Maven property using the latest release version (tag) of a GitHub repository
+
+    env:
+      DEP_PROP: hugo-geekdoc.version
+      DEP_REPO: thegeeklab/hugo-geekdoc
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Setup Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: 17
+          cache: 'maven'
+
+      - name: Get latest release version
+        id: lr
+        uses: pozetroninc/github-action-get-latest-release@master
+        with:
+          repository: ${{ env.DEP_REPO }}
+
+      - name: Apply latest release version on Maven property
+        run: >
+          mvn -B -ntp versions:set-property 
+          -Dproperty=${{ env.DEP_PROP }}
+          -DnewVersion=${{ steps.lr.outputs.release }} 
+          -DgenerateBackupPoms=false
+
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v5
+        with:
+          commit-message: Bump ${{ env.DEP_PROP }} to ${{ steps.lr.outputs.release }}
+          title: Bump ${{ env.DEP_PROP }} to ${{ steps.lr.outputs.release }}
+          body: ${{ steps.lr.outputs.description }}
+          delete-branch: true
+          branch: PR/maven/${{ github.ref_name }}/${{ env.DEP_PROP }}-${{ steps.lr.outputs.release }}
+          labels: dependencies,java
+
+      - name: Create summary
+        if: ${{ steps.cpr.outputs.pull-request-number }}
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" >> $GITHUB_STEP_SUMMARY
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/java-ea-maven.yml b/.github/workflows/java-ea-maven.yml
@@ -11,15 +11,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ '-ea' ]
+        java: [ 20 ]
         os: [ ubuntu-latest ]
 
     name: JDK${{ matrix.java }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 0

diff --git a/.github/workflows/java8-maven.yml b/.github/workflows/java8-maven.yml
@@ -4,7 +4,7 @@ on: [ push, pull_request ]
 
 jobs:
   build-and-test-job:
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/heads/develop') || startsWith(github.ref, 'refs/tags/v')
     strategy:
       fail-fast: false
       matrix:
@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 0
@@ -28,11 +28,31 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
 
+      - name: Pre-download dependencies with Maven
+        run: mvn -U -B -ntp dependency:go-offline
+
       - name: Build and (headless) test with Maven
         uses: smithki/xvfb-action@v1.1.2
         with:
           run: mvn -U -B -ntp package
 
+  auto-merge-job:
+    needs: build-and-test-job
+    if: startsWith(github.repository, 'nbbrd/') && github.event_name == 'pull_request' && github.actor == 'dependabot[bot]'
+    permissions:
+      contents: write
+      pull-requests: write
+
+    name: Auto-merge on dependabot PR
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Merge PR
+        run: gh pr merge --auto --rebase "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
   snapshot-job:
     needs: build-and-test-job
     if: startsWith(github.repository, 'nbbrd/') && startsWith(github.ref, 'refs/heads/develop')
@@ -46,7 +66,7 @@ jobs:
 
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 0
@@ -96,7 +116,7 @@ jobs:
 
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 0

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,48 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [3.0.0-beta.12] - 2023-10-16
+
+This is the twelfth beta release of **sdmx-dl**.  
+sdmx-dl follows [semantic versioning](https://semver.org/).
+
+This release brings a more flexible configuration system: cache, network and language can be specified by source if
+needed and environment variables are used as default values.  
+The API has been refactored to make it both simpler and more flexible.  
+The ECB endpoint has also been updated.
+
+> **Disclaimer**: sdmx-dl is still in development and is available <ins>for test only</ins>. **Do not use in
+production!**
+
+### Added
+
+- ![API] Add support of partial ISO date/time in TimeInterval
+- ![PROVIDER] Allow caching & networking configuration by environment
+  variables [#516](https://github.com/nbbrd/sdmx-dl/issues/516)
+- ![PROVIDER] Allow caching & networking configuration per source [#493](https://github.com/nbbrd/sdmx-dl/issues/493)
+
+### Fixed
+
+- ![PROVIDER] Fix file locking in cache
+
+### Changed
+
+- ![API] Refactor cache API [#500](https://github.com/nbbrd/sdmx-dl/issues/500)
+- ![API] Refactor network API [#503](https://github.com/nbbrd/sdmx-dl/issues/503)
+- ![API] Remove dialect API [#505](https://github.com/nbbrd/sdmx-dl/issues/505)
+- ![API] Refactor listeners API [#506](https://github.com/nbbrd/sdmx-dl/issues/506)
+- ![API] Handle languages per connection instead of per manager [#507](https://github.com/nbbrd/sdmx-dl/issues/507)
+- ![API] Simplify naming and structure [#508](https://github.com/nbbrd/sdmx-dl/issues/508)
+- ![API] Change pattern of drivers properties
+- ![API] Improve support of ISO-8601 duration
+- ![SOURCE] Update ECB endpoint [#495](https://github.com/nbbrd/sdmx-dl/issues/495)
+- ![FORMAT] Refactor file format API [#502](https://github.com/nbbrd/sdmx-dl/issues/502)
+- ![PROVIDER] Move curl backend to an external project
+- ![PROVIDER] Move PxWebDriver to its own module
+- ![PROVIDER] Move dialects drivers to their own module
+- ![CLI] Use environment variables as default values
+- ![GRPC] Migrate gRPC module to Quarkus framework
+
 ## [3.0.0-beta.11] - 2023-06-02
 
 This is the eleventh beta release of **sdmx-dl**.  
@@ -377,7 +419,9 @@ production._
 
 - Initial release
 
-[Unreleased]: https://github.com/nbbrd/sdmx-dl/compare/v3.0.0-beta.11...HEAD
+[Unreleased]: https://github.com/nbbrd/sdmx-dl/compare/v3.0.0-beta.12...HEAD
+
+[3.0.0-beta.12]: https://github.com/nbbrd/sdmx-dl/compare/v3.0.0-beta.11...v3.0.0-beta.12
 
 [3.0.0-beta.11]: https://github.com/nbbrd/sdmx-dl/compare/v3.0.0-beta.10...v3.0.0-beta.11
 

diff --git a/docs/content/api/_index.md b/docs/content/api/_index.md
@@ -3,9 +3,11 @@ title: "Java library"
 weight: 1
 ---
 
-_work-in-progress_
+![_work-in-progress_](https://img.shields.io/badge/-work_in_progress-E2BC4A)
 
-## Dependency graph
+**sdmx-dl library** is designed as a facade for the SDMX model and APIs.
+
+## Structure overview
 
 ```mermaid
 flowchart BT
@@ -21,12 +23,60 @@ flowchart BT
         provider-base[base]
         connectors
         ri
+        dialects
+        px
+    end
+    subgraph applications
+        cli
+        desktop
+        grpc
     end
     testing
-    cli
 
     formats --> api
     providers --> formats
     testing ---> api
-    cli --> providers & testing
+    applications --> providers & testing
+```
+
+## Dependencies setup
+
+sdmx-dl is **distributed in two flavours**: a **standard JAR hierarchy**, and a **standalone uber JAR** containing all the implementations and their dependencies.
+Most of the standalone JAR’s dependencies are shaded i.e. they are hidden in alternative packages.
+This allows sdmx-dl to be used in projects with conflicting versions of its dependencies.
+
+Standard JAR hierarchy:
+
+```xml
+<dependencies>
+  <dependency>
+    <groupId>com.github.nbbrd.sdmx-dl</groupId>
+    <artifactId>sdmx-dl-api</artifactId>
+  </dependency>
+  <dependency>
+    <groupId>com.github.nbbrd.sdmx-dl</groupId>
+    <artifactId>sdmx-dl-ri</artifactId>
+    <scope>runtime</scope>
+  </dependency>
+  <dependency>
+    <groupId>com.github.nbbrd.sdmx-dl</groupId>
+    <artifactId>sdmx-dl-dialects</artifactId>
+    <scope>runtime</scope>
+  </dependency>
+</dependencies>
+```
+
+Standalone uber JAR:
+
+```xml
+<dependencies>
+  <dependency>
+    <groupId>com.github.nbbrd.sdmx-dl</groupId>
+    <artifactId>sdmx-dl-api</artifactId>
+  <dependency>
+    <groupId>com.github.nbbrd.sdmx-dl</groupId>
+    <artifactId>sdmx-dl-standalone</artifactId>
+    <scope>runtime</scope>
+  </dependency>
+</dependencies>
 ```
diff --git a/docs/content/cli/options.md b/docs/content/cli/options.md
@@ -49,12 +49,12 @@ The format is [RFC4180](https://tools.ietf.org/html/rfc4180).
 
 Network-related options used to deal with performance, proxies, security and authentication.
 
-| Name                                                                 | Shortcut | Parameter                              | Description                                                     |
-|----------------------------------------------------------------------|----------|----------------------------------------|-----------------------------------------------------------------|
-| <a id="curl" href="#curl">`--curl`</a>                               | -        | -                                      | Use curl backend instead of JDK.                                |
-| <a id="no-cache" href="#no-cache">`--no-cache`</a>                   | -        | -                                      | Disable caching.                                                |
-| <a id="auto-proxy" href="#auto-proxy">`--auto-proxy`</a>             | -        | -                                      | Enable automatic proxy detection.                               |
-| <a id="no-default-ssl" href="#no-default-ssl">`--no-default-ssl`</a> | -        | -                                      | Disable default truststore.                                     |
-| <a id="no-system-ssl" href="#no-system-ssl">`--no-system-ssl`</a>    | -        | -                                      | Disable system truststore.                                      |
-| <a id="no-system-auth" href="#no-system-auth">`--no-system-auth`</a> | -        | -                                      | Disable system authentication.                                  |
-| <a id="user" href="#user">`--user`</a>                               | -        | [`<user:password>`](../datatypes#user) | Specify the user and password to use for server authentication. |
+| Name                                                                   | Shortcut | Parameter                              | Description                                                     |
+|------------------------------------------------------------------------|----------|----------------------------------------|-----------------------------------------------------------------|
+| <a id="auto-proxy" href="#auto-proxy">`--[no-]auto-proxy`</a>          | -        | -                                      | Enable automatic proxy detection.                               |
+| <a id="curl" href="#curl">`--[no-]curl`</a>                            | -        | -                                      | Use curl backend instead of JDK.                                |
+| <a id="no-default-ssl" href="#no-default-ssl">`--[no-]default-ssl`</a> | -        | -                                      | Disable default truststore.                                     |
+| <a id="no-system-ssl" href="#no-system-ssl">`--[no-]system-ssl`</a>    | -        | -                                      | Disable system truststore.                                      |
+| <a id="no-cache" href="#no-cache">`--no-cache`</a>                     | -        | -                                      | Disable caching.                                                |
+| <a id="no-system-auth" href="#no-system-auth">`--system-auth`</a>      | -        | -                                      | Disable system authentication.                                  |
+| <a id="user" href="#user">`--user`</a>                                 | -        | [`<user:password>`](../datatypes#user) | Specify the user and password to use for server authentication. |
diff --git a/docs/content/cli/usage.md b/docs/content/cli/usage.md
@@ -243,7 +243,6 @@ Output format:
     [`Description:string`](../datatypes#string),
     [`Aliases:list`](../datatypes#list),
     [`Driver:string`](../datatypes#string),
-    [`Dialect:string`](../datatypes#string),
     [`Endpoint:uri`](../datatypes#uri),
     [`Properties:map`](../datatypes#list),
     [`Website:url`](../datatypes#url),
@@ -254,11 +253,11 @@ Output format:
 
 {{< expand "Output sample" >}}
 
-| Name  | Description                     | Aliases  | Driver              | Dialect | Endpoint                                         | Properties           | Website                                     | Monitor                           | MonitorWebsite                                     | Languages |
-|-------|---------------------------------|----------|---------------------|---------|--------------------------------------------------|----------------------|---------------------------------------------|-----------------------------------|----------------------------------------------------|-----------|
-| ABS   | Australian Bureau of Statistics |          | ri:abs              |         | https://stat.data.abs.gov.au/restsdmx/sdmx.ashx  |                      | https://stat.data.abs.gov.au                | upptime:/nbbrd/sdmx-upptime/ABS   | https://nbbrd.github.io/sdmx-upptime/history/abs   | en        |
-| ECB   | European Central Bank           |          | ri:sdmx21           | ECB2020 | https://sdw-wsrest.ecb.europa.eu/service         | detailSupported=true | https://sdw.ecb.europa.eu                   | upptime:/nbbrd/sdmx-upptime/ECB   | https://nbbrd.github.io/sdmx-upptime/history/ecb   | en        |
-| ESTAT | Eurostat                        | EUROSTAT | connectors:eurostat |         | https://ec.europa.eu/eurostat/SDMX/diss-web/rest |                      | https://ec.europa.eu/eurostat/data/database | upptime:/nbbrd/sdmx-upptime/ESTAT | https://nbbrd.github.io/sdmx-upptime/history/estat | en,de,fr  |
+| Name  | Description                     | Aliases  | Driver              | Endpoint                                         | Properties           | Website                                     | Monitor                           | MonitorWebsite                                     | Languages |
+|-------|---------------------------------|----------|---------------------|--------------------------------------------------|----------------------|---------------------------------------------|-----------------------------------|----------------------------------------------------|-----------|
+| ABS   | Australian Bureau of Statistics |          | ri:abs              | https://stat.data.abs.gov.au/restsdmx/sdmx.ashx  |                      | https://stat.data.abs.gov.au                | upptime:/nbbrd/sdmx-upptime/ABS   | https://nbbrd.github.io/sdmx-upptime/history/abs   | en        |
+| ECB   | European Central Bank           |          | ri:sdmx21           | https://sdw-wsrest.ecb.europa.eu/service         | detailSupported=true | https://sdw.ecb.europa.eu                   | upptime:/nbbrd/sdmx-upptime/ECB   | https://nbbrd.github.io/sdmx-upptime/history/ecb   | en        |
+| ESTAT | Eurostat                        | EUROSTAT | connectors:eurostat | https://ec.europa.eu/eurostat/SDMX/diss-web/rest |                      | https://ec.europa.eu/eurostat/data/database | upptime:/nbbrd/sdmx-upptime/ESTAT | https://nbbrd.github.io/sdmx-upptime/history/estat | en,de,fr  |
 
 {{< /expand >}}
 

diff --git a/docs/layouts/shortcodes/sources.html b/docs/layouts/shortcodes/sources.html
@@ -14,9 +14,9 @@
     <tr>
         {{ $name := index $r 0 }}
         {{ $description := index $r 1 }}
-        {{ $website := index $r 7 }}
+        {{ $website := index $r 6 }}
         {{ $domain := (urls.Parse $website).Host }}
-        {{ $monitor_website := index $r 9 }}
+        {{ $monitor_website := index $r 8 }}
         {{ $monitor_name := replace (lower $name) "_" "-" }}
         <td><code>{{ $name }}</code></td>
         <td><img src="https://www.google.com/s2/favicons?domain={{ $domain }}" /> <a href="{{ $website }}">{{ $description }}</a></td>