Merge pull request #4 from rok4/develop

Realease 1.1.0
rok4 · Sep 12, 2023 · 3bc9113 · 3bc9113
2 parents 12c389f + a2fa3fd
commit 3bc9113
Show file tree

Hide file tree

Showing 23 changed files with 444 additions and 688 deletions.
diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml
@@ -0,0 +1,54 @@
+name: Build and release
+
+on:
+  push:
+    tags:
+      - '**'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Organize files
+        run: |
+          perl Makefile.PL DESTDIR=build INSTALL_BASE=/usr
+          make
+          make pure_install
+
+      - name: Build debian package
+        id: build_debian
+        uses: jiro4989/build-deb-action@v2
+        with:
+          package: librok4-core-perl
+          package_root: build
+          maintainer: Géoportail<tout_rdev@ign.fr>
+          version: ${{ github.ref_name }}
+          arch: 'all'
+          desc: 'Perl core libraries are used by ROK4 pregeneration and managment tools'
+          depends: perl-base, libgdal-perl, libpq-dev, gdal-bin, libfile-find-rule-perl, libfile-copy-link-perl, libconfig-ini-perl, libdbi-perl, libdbd-pg-perl, libdevel-size-perl, libdigest-sha-perl, libfile-map-perl, libfindbin-libs-perl, libhttp-message-perl, liblwp-protocol-https-perl, libmath-bigint-perl, libterm-progressbar-perl, liblog-log4perl-perl, libjson-parse-perl, libjson-perl, libjson-validator-perl, libtest-simple-perl, libxml-libxml-perl, libnet-amazon-s3-perl
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref_name }}
+          body_path: CHANGELOG.md
+          draft: false
+          prerelease: false
+
+      - name: Add debian package to release
+        id: upload-release-asset 
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./${{ steps.build_debian.outputs.file_name }}
+          asset_name: librok4-core-perl-${{ github.ref_name }}-ubuntu-20.04-all.deb
+          asset_content_type: application/vnd.debian.binary-package
+
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@ Makefile
 MYMETA.*
 blib
 pm_to_blib
+build
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,23 @@
+## Changelog
+
+### [Fixed]
+
+* Indentification de la compression dans les formats "RAW"
+
+### [Added]
+
+* Possibilité de préciser une région S3 et si on souhaite utiliser des hôtes virtuels (bucket en sous domaine)
+
+<!-- 
+### [Added]
+
+### [Changed]
+
+### [Deprecated]
+
+### [Removed]
+
+### [Fixed]
+
+### [Security] 
+-->
diff --git a/Makefile.PL b/Makefile.PL
@@ -7,10 +7,10 @@ WriteMakefile(
     BUILD_REQUIRES => { 
         "ExtUtils::MakeMaker" => 0
     },
-    PREREQ_FATAL => 1,
     # ack -h "^use [A-Z]" ./ | cut -d' ' -f2 | grep -v -E "ROK4::" | sed -r "s#;##" | sort | uniq | sed -r "s/(.+)/\"\1\" => 0,/"
     PREREQ_PM => {
-        "Amazon::S3" => 0,
+        "Net::Amazon::S3" => 0,
+        "Net::Amazon::S3::Vendor::Generic" => 0,
         "Config::INI::Reader" => 0,
         "Cwd" => 0,
         "Data::Dumper" => 0,

diff --git a/README.md b/README.md
@@ -1,46 +1,49 @@
 # Librairies CORE Perl
 
-- [Dépendances](#dépendances)
-- [Installation](#installation)
-- [Utilisation en submodule GIT](#utilisation-en-submodule-git)
-- [Variables d'environnement utilisées dans les librairies](#variables-denvironnement-utilisées-dans-les-librairies)
+Ces librairies Perl sont utilisées par les outils de [prégénération](https://github.com/rok4/pregeneration) et ceux de [gestion et analyse](https://github.com/rok4/tools)
 
-## Dépendances
+## Installation depuis le paquet debian
 
-* Paquets debian
-    * perl-base
-    * libgdal-perl
-    * libpq-dev
-    * gdal-bin
-    * libfile-find-rule-perl
-    * libfile-copy-link-perl
-    * libconfig-ini-perl
-    * libdbi-perl
-    * libdbd-pg-perl
-    * libdevel-size-perl
-    * libdigest-sha-perl
-    * libfile-map-perl
-    * libfindbin-libs-perl
-    * libhttp-message-perl
-    * liblwp-protocol-https-perl
-    * libmath-bigint-perl
-    * libterm-progressbar-perl
-    * liblog-log4perl-perl
-    * libjson-parse-perl
-    * libjson-perl
-    * libjson-validator-perl
-    * libtest-simple-perl
-    * libxml-libxml-perl
-    * libamazon-s3-perl
+Télécharger le paquet sur GitHub : https://github.com/rok4/core-perl/releases/
 
-## Installation
+```
+apt install ./librok4-core-perl-<version>-linux-all.deb
+```
 
-`perl Makefile.PL INSTALL_BASE=/usr/local VERSION=0.0.1`
+## Installation depuis les sources
 
-## Utilisation en submodule GIT
+Dépendances (paquets debian) :
 
-* Si le dépôt de code est à côté : `git submodule add ../core-perl.git core`
-* Sinon : `git submodule add https://github.com/rok4/core-perl.git core`
+* perl-base
+* libgdal-perl
+* libpq-dev
+* gdal-bin
+* libfile-find-rule-perl
+* libfile-copy-link-perl
+* libconfig-ini-perl
+* libdbi-perl
+* libdbd-pg-perl
+* libdevel-size-perl
+* libdigest-sha-perl
+* libfile-map-perl
+* libfindbin-libs-perl
+* libhttp-message-perl
+* liblwp-protocol-https-perl
+* libmath-bigint-perl
+* libterm-progressbar-perl
+* liblog-log4perl-perl
+* libjson-parse-perl
+* libjson-perl
+* libjson-validator-perl
+* libtest-simple-perl
+* libxml-libxml-perl
+* libnet-amazon-s3-perl
+
+```
+perl Makefile.PL INSTALL_BASE=/usr PREREQ_FATAL=1
+make
+make install
+```
 
 ## Variables d'environnement utilisées dans les librairies
 
@@ -55,6 +58,8 @@ Leur définition est contrôlée à l'usage.
     - `ROK4_S3_URL`
     - `ROK4_S3_KEY`
     - `ROK4_S3_SECRETKEY`
+    - (optionnel) `ROK4_S3_REGION`
+    - (optionnel) `ROK4_S3_USE_VIRTUAL_HOST`
 * Pour le stockage SWIFT
     - `ROK4_SWIFT_AUTHURL`
     - `ROK4_SWIFT_USER`

diff --git a/lib/ROK4/Core/Array.pm b/lib/ROK4/Core/Array.pm
@@ -40,7 +40,7 @@ File: Array.pm
 
 Class: ROK4::Core::Array
 
-(see libperlauto/Core_Array.png)
+(see libperlauto/ROK4_Core_Array.png)
 
 Provides functions to compute the max value or the min index of an array. Do not instanciate.
 

diff --git a/lib/ROK4/Core/Base36.pm b/lib/ROK4/Core/Base36.pm
@@ -40,7 +40,7 @@ File: Base36.pm
 
 Class: ROK4::Core::Base36
 
-(see libperlauto/Core_Base36.png)
+(see libperlauto/ROK4_Core_Base36.png)
 
 Base 36 converting tools. Do not instanciate.
 

diff --git a/lib/ROK4/Core/Database.pm b/lib/ROK4/Core/Database.pm
@@ -40,7 +40,7 @@ File: Database.pm
 
 Class: ROK4::Core::Database
 
-(see libperlauto/Core_Database.png)
+(see libperlauto/ROK4_Core_Database.png)
 
 Allow to request a PostgreSQL database.
 
@@ -54,11 +54,6 @@ Using:
 
 Attributes:
     id - string - Connection identifier, concatenation of following values, used to caching results.
-    dbname - string - database name
-    host - string - PostgreSQL server host
-    port - integer - PostgreSQL server port
-    username - string - PostgreSQL server user
-    password - string - PostgreSQL server user's password
 
     connection - DBI database - PostgreSQL connection, use to execute requests
 
@@ -123,15 +118,7 @@ sub new {
 
     my $this = {
         id => "$dbname,$host,$port,$username,$password",
-
-        dbname => $dbname,
-        host => $host,
-        port => $port,
-        username => $username,
-        password => $password,
-
         connection => undef,
-
         current_results => undef
     };
 
@@ -997,6 +984,7 @@ Parameter (list):
     schema_name - string - Schema in which determine min and max
     table_name - string - Table in which determine min and max
     att_name - string - Attribute for which we want to determine min and max
+    filter - string - Optionnal, to limit considered tuples
 
 Return
     An array (min, max)
@@ -1006,16 +994,23 @@ sub get_min_max_values {
     my $schema_name = shift;
     my $table_name = shift;
     my $att_name = shift;
+    my $filter = shift;
 
-    if (exists $CACHE->{$this->{id}}->{get_min_max_values}->{$schema_name}->{$table_name}->{$att_name}) {
-        return @{$CACHE->{$this->{id}}->{get_min_max_values}->{$schema_name}->{$table_name}->{$att_name}};
+    my $sql;
+    if (defined $filter && $filter ne "") {
+        $sql = "SELECT min($att_name), max($att_name) FROM $schema_name.$table_name WHERE $filter;"
+    } else {
+        $sql = "SELECT min($att_name), max($att_name) FROM $schema_name.$table_name;";
+        $filter = "none";
     }
 
-    my $sql = sprintf "SELECT min($att_name), max($att_name) FROM $schema_name.$table_name;";
+    if (exists $CACHE->{$this->{id}}->{get_min_max_values}->{$schema_name}->{$table_name}->{$att_name}->{$filter}) {
+        return @{$CACHE->{$this->{id}}->{get_min_max_values}->{$schema_name}->{$table_name}->{$att_name}->{$filter}};
+    }
 
     my @line = $this->select_one_row($sql);
 
-    $CACHE->{$this->{id}}->{get_min_max_values}->{$schema_name}->{$table_name}->{$att_name} = [$line[0], $line[1]];
+    $CACHE->{$this->{id}}->{get_min_max_values}->{$schema_name}->{$table_name}->{$att_name}->{$filter} = [$line[0], $line[1]];
     return ($line[0], $line[1]);
 
 }
@@ -1027,6 +1022,7 @@ Parameter (list):
     schema_name - string - Schema in which count distinct values
     table_name - string - Table in which count distinct values
     att_name - string - Attribute for which we want to count distinct values
+    filter - string - Optionnal, to limit considered tuples
 
 Return
     Distinct values count
@@ -1036,16 +1032,23 @@ sub get_distinct_values_count {
     my $schema_name = shift;
     my $table_name = shift;
     my $att_name = shift;
+    my $filter = shift;
 
-    if (exists $CACHE->{$this->{id}}->{get_distinct_values_count}->{$schema_name}->{$table_name}->{$att_name}) {
-        return $CACHE->{$this->{id}}->{get_distinct_values_count}->{$schema_name}->{$table_name}->{$att_name};
+    my $sql;
+    if (defined $filter && $filter ne "") {
+        $sql = "SELECT count(*) FROM (SELECT DISTINCT $att_name FROM $schema_name.$table_name WHERE $filter) as tmp;"
+    } else {
+        $sql = "SELECT count(*) FROM (SELECT DISTINCT $att_name FROM $schema_name.$table_name) as tmp;";
+        $filter = "none";
     }
 
-    my $sql = sprintf "SELECT count(*) FROM (SELECT DISTINCT $att_name FROM $schema_name.$table_name) as tmp;";
+    if (exists $CACHE->{$this->{id}}->{get_distinct_values_count}->{$schema_name}->{$table_name}->{$att_name}->{$filter}) {
+        return $CACHE->{$this->{id}}->{get_distinct_values_count}->{$schema_name}->{$table_name}->{$att_name}->{$filter};
+    }
 
     my @line = $this->select_one_row($sql);
 
-    $CACHE->{$this->{id}}->{get_distinct_values_count}->{$schema_name}->{$table_name}->{$att_name} = $line[0];
+    $CACHE->{$this->{id}}->{get_distinct_values_count}->{$schema_name}->{$table_name}->{$att_name}->{filter} = $line[0];
     return $line[0];
 }
 
@@ -1057,6 +1060,7 @@ Parameter (list):
     schema_name - string - Schema in which list distinct values
     table_name - string - Table in which list distinct values
     att_name - string - Attribute for which we want to list distinct values
+    filter - string - Optionnal, to limit considered tuples
 
 Return
     Distinct values in an array, an empty array if failure
@@ -1066,14 +1070,22 @@ sub get_distinct_values {
     my $schema_name = shift;
     my $table_name = shift;
     my $att_name = shift;
+    my $filter = shift;
 
-    if (exists $CACHE->{$this->{id}}->{get_distinct_values}->{$schema_name}->{$table_name}->{$att_name}) {
-        return @{$CACHE->{$this->{id}}->{get_distinct_values}->{$schema_name}->{$table_name}->{$att_name}};
+    my $sql;
+    if (defined $filter && $filter ne "") {
+        $sql = "SELECT DISTINCT $att_name FROM $schema_name.$table_name WHERE $filter;"
+    } else {
+        $sql = "SELECT DISTINCT $att_name FROM $schema_name.$table_name;";
+        $filter = "none";
     }
 
-    my $sql = sprintf "SELECT DISTINCT $att_name FROM $schema_name.$table_name;";
+    if (exists $CACHE->{$this->{id}}->{get_distinct_values}->{$schema_name}->{$table_name}->{$att_name}->{$filter}) {
+        return @{$CACHE->{$this->{id}}->{get_distinct_values}->{$schema_name}->{$table_name}->{$att_name}->{$filter}};
+    }
 
     my $att_values = $this->select_all_row($sql);
+
     if ( ! defined $att_values) {
         ERROR("Cannot list distinct values of $att_name in $schema_name.$table_name");
         return ();
@@ -1087,7 +1099,7 @@ sub get_distinct_values {
         }
     }
 
-    $CACHE->{$this->{id}}->{get_distinct_values}->{$schema_name}->{$table_name}->{$att_name} = \@array;
+    $CACHE->{$this->{id}}->{get_distinct_values}->{$schema_name}->{$table_name}->{$att_name}->{$filter} = \@array;
     return @array;
 }
 

diff --git a/lib/ROK4/Core/GeoImage.pm b/lib/ROK4/Core/GeoImage.pm
@@ -40,7 +40,7 @@ File: GeoImage.pm
 
 Class: ROK4::Core::GeoImage
 
-(see libperlauto/Core_GeoImage.png)
+(see libperlauto/ROK4_Core_GeoImage.png)
 
 Describes a georeferenced image and enable to know its components.
 

diff --git a/lib/ROK4/Core/GeoVector.pm b/lib/ROK4/Core/GeoVector.pm
@@ -40,7 +40,7 @@ File: GeoVector.pm
 
 Class: ROK4::Core::GeoVector
 
-(see libperlauto/Core_GeoVector.png)
+(see libperlauto/ROK4_Core_GeoVector.png)
 
 Describes a georeferenced image and enable to know its components.
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,3 +2,4 @@ Makefile @@
     MYMETA.*
     blib
     pm_to_blib
+    build