Merge pull request #61 from mjanez/ckan-2.9.9

Update plugins and image fixes (nginx, dev)

.env.example

@@ -118,7 +118,7 @@ CKAN__LOCALE_DEFAULT="en"
 CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv"
 
 # Extensions
-CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface scheming_datasets scheming_groups scheming_organizations pdf_view pages facet_scheming"
+CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface scheming_dcat_datasets scheming_dcat_groups scheming_dcat_organizations scheming_dcat pdf_view pages"
 
 # ckanext-harvest
 CKAN__HARVEST__MQ__TYPE=redis
@@ -134,12 +134,6 @@ CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckan:ckan@db/ckan
 CKANEXT__DCAT__BASE_URI=http://localhost/catalog
 CKANEXT__DCAT__RDF_PROFILES="euro_dcat_ap_2 euro_dcat_ap"
 
-# ckanext-scheming: setup_scheming.sh
-SCHEMA_CKANEXT_SCHEMING_DATASET_SCHEMA="ckanext.scheming:ckan_geodcatap.yaml"
-SCHEMA_CKANEXT_SCHEMING_GROUP_SCHEMAS="ckanext.scheming:ckan_group_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_ORGANIZATION_SCHEMAS="ckanext.scheming:ckan_org_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_PRESETS="ckanext.scheming:presets.json"
-
 # ckanext-spatial (Solr Backend - solr8-spatial)
 CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox
 CKAN__SPATIAL__SRID=3857
@@ -153,5 +147,17 @@ CKANEXT__GEOVIEW__OL_VIEWER__FORMATS="wms wfs geojson gml kml"
 CKANEXT__GEOVIEW__SHP_VIEWER__SRID=3857
 CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8
 
-# ckanext-facet_scheming
-CKANEXT__FACET_FACET_LIST="theme theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to "
+# ckanext-scheming_dcat
+## Scheming: setup_scheming.sh
+CKANEXT__SCHEMING_DCAT_DATASET_SCHEMA="ckanext.scheming_dcat:schemas/geodcatap/geodcatap_dataset.yaml"
+CKANEXT__SCHEMING_DCAT_GROUP_SCHEMAS="ckanext.scheming_dcat:schemas/geodcatap/geodcatap_group.json"
+CKANEXT__SCHEMING_DCAT_ORGANIZATION_SCHEMAS="ckanext.scheming_dcat:schemas/geodcatap/geodcatap_org.json"
+CKANEXT__SCHEMING_DCAT_PRESETS="ckanext.scheming_dcat:schemas/geodcatap/geodcatap_presets.json"
+## Facets: setup_scheming.sh
+CKANEXT__SCHEMING_DCAT_FACET_LIST="theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to"
+CKANEXT__SCHEMING_DCAT_ORGANIZATION_CUSTOM_FACETS=True
+CKANEXT__SCHEMING_DCAT_GROUP_CUSTOM_FACETS=True
+
+# WIP: ckanext-sparql_interface
+CKANEXT__SPARQL__ENDPOINT_URL=https://dbpedia.org/sparql
+CKANEXT__SPARQL__HIDE_ENDPOINT_URL=False

README.md

@@ -82,7 +82,7 @@ Information about extensions installed in the `main` image. More info described
 | Extension   | [ckanext-resourcedictionary](https://github.com/OpenDataGIS/ckanext-resourcedictionary) | main        | Completed                    | ✔️      | ✔️      | Stable installation. This extension extends the default CKAN Data Dictionary functionality by adding possibility to create data dictionary before actual data is uploaded to datastore.                                                                                                                                                                                 |
 | Extension   | [ckanext-pages](https://github.com/ckan/ckanext-pages)                                  | 0.5.1       | Completed                    | ✔️      | ✔️      | Stable installation. This extension gives you an easy way to add simple pages to CKAN.                                                                                                                                                                                                                                                                                  |
 | Extension   | [ckanext-pdfview](https://github.com/ckan/ckanext-pdfview)                              | 0.0.8       | Completed                    | ✔️      | ✔️      | Stable installation. This extension provides a view plugin for PDF files using an html object tag.                                                                                                                                                                                                                                                                      |
-| Extension    | [ckanext-facet_scheming](https://github.com/OpenDataGIS/ckanext-facet_scheming)                                    | 1.0.0        | Completed | ✔️      | ✔️       | Stable installation for 1.0.0 version, facet and filter for custom [ckanext-scheming](https://github.com/mjanez/ckanext-scheming)                                                                                                                                                                         |
+| Extension    | [ckanext-scheming_dcat](https://github.com/OpenDataGIS/ckanext-scheming_dcat)                                    | 1.0.0        | Completed | ✔️      | ✔️       | Stable installation for 1.0.0 version, DCAT improved, facet and filter for custom [ckanext-scheming](https://github.com/mjanez/ckanext-scheming)                                                                                                                                                                         |
 | Software    | [ckan-pycsw](https://github.com/mjanez/ckan-pycsw)                                    | latest        | Completed | ✔️      | ✔️       | Stable installation. PyCSW Endpoint of Open Data Portal with docker compose config. Harvest the CKAN catalogue in a CSW endpoint based on existing spatial datasets in the open data portal.                                                                                                                                                                            |
 
 
@@ -437,7 +437,7 @@ PostgreSQL offers the command line tools [`pg_dump`](https://www.postgresql.org/
     POSTGRES_USER="ckan"
     POSTGRES_PASSWORD="your_postgres_password"
     BACKUP_DIRECTORY="/path/to/your/backup/directory"
-    DATE=`date +%Y%m%d%H%M%S`
+    DATE=$(date +%Y%m%d%H%M%S)
 
     # Run the backup command
     docker exec -e PGPASSWORD=$POSTGRES_PASSWORD $POSTGRESQL_CONTAINER_NAME pg_dump -U $POSTGRES_USER -Fc $DATABASE_NAME > $BACKUP_DIRECTORY/ckan_backup_$DATE.dump
@@ -477,6 +477,9 @@ PostgreSQL offers the command line tools [`pg_dump`](https://www.postgresql.org/
 
 The cronjob is now set up and will backup your CKAN PostgreSQL database daily at midnight using the custom format. The backups will be stored in the specified directory with the timestamp in the filename.
 
+>**Info**<br>
+> Sample scripts for backing up CKAN: [`doc/scripts`](doc/scripts)
+
 ### Restore a backup
 If need to use a backup, restore it:
 

ckan/Dockerfile

@@ -13,11 +13,11 @@ COPY req_fixes ${APP_DIR}/req_fixes
 ### Geoview - v0.0.20 ###
 ### Spatial - v2.0.0 ### fixed requirements.txt
 ### DCAT - 1.0.0-geodcatap (GeoDCAT-AP extended version) ###
-### Scheming - 1.1.0-geodcatap (GeoDCAT-AP extended version) ###
+### Scheming - release-3.0.0 ###
 ### Resource dictionary - 1.0.1 ###
 ### Pages - v0.5.2 ###
 ### PDFView - 0.0.8 ###
-### Facet Scheming - 1.0.1 ###
+### Scheming DCAT - v1.0.0 (GeoDCAT-AP extended version) ###
 ### SPARQL Interface - 2.0.1 ###
 RUN echo ${TZ} > /etc/timezone && \
     if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime ; fi && \
@@ -37,22 +37,22 @@ RUN echo ${TZ} > /etc/timezone && \
     echo "mjanez/ckanext-dcat (GeoDCAT-AP extended version)" && \
     pip3 install -e git+https://github.com/mjanez/ckanext-dcat.git@1.0.0-geodcatap#egg=ckanext-dcat && \
     pip3 install -r https://raw.githubusercontent.com/mjanez/ckanext-dcat/master/requirements.txt && \
-    echo "mjanez/ckanext-scheming (GeoDCAT-AP extended version)" && \
-    pip3 install -e git+https://github.com/mjanez/ckanext-scheming.git@1.1.0-geodcatap#egg=ckanext-scheming && \
+    echo "ckan/ckanext-scheming" && \
+    pip3 install -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming && \
     echo "mjanez/ckanext-resourcedictionary" && \
     pip3 install -e 'git+https://github.com/mjanez/ckanext-resourcedictionary.git@1.0.1#egg=ckanext-resourcedictionary' && \
     echo "ckan/ckanext-pages" && \
     pip3 install -e git+https://github.com/ckan/ckanext-pages.git@v0.5.2#egg=ckanext-pages && \
     echo "ckan/ckanext-pdfview" && \
     pip3 install -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.8#egg=ckanext-pdfview && \
-    echo "OpenDataGIS/ckanext-facet_scheming" && \
-    pip3 install -e git+https://github.com/OpenDataGIS/ckanext-facet_scheming.git@1.0.1#egg=ckanext_facet_scheming && \
-    pip3 install -r https://raw.githubusercontent.com/OpenDataGIS/ckanext-facet_scheming/1.0.1/requirements.txt
+    echo "OpenDataGIS/ckanext-scheming_dcat" && \
+    pip3 install -e git+https://github.com/mjanez/ckanext-scheming_dcat.git@v1.0.0#egg=ckanext_scheming_dcat && \
+    pip3 install -r https://raw.githubusercontent.com/mjanez/ckanext-scheming_dcat/v1.0.0/requirements.txt
 
 # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc
 COPY docker-entrypoint.d/* /docker-entrypoint.d/
 
-# Update who.ini with PROXY_CKAN_LOCATION
+# Update who.ini with APACHE_CKAN_LOCATION
 COPY setup/who.ini ${APP_DIR}/
 
 # Apply any patches needed to CKAN core

ckan/Dockerfile.dev

@@ -6,6 +6,7 @@ ENV APP_DIR=/srv/app \
     SRC_EXTENSIONS_DIR=/srv/app/src_extensions
 
 RUN echo ${TZ} > /etc/timezone && \
+    set -ex && apk --no-cache add sudo && \
     # Make sure both files are not exactly the same
     if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime ; fi
 
@@ -57,12 +58,12 @@ RUN chmod +x ${APP_DIR}/start_ckan_development.sh
 
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
-# COPY patches ${APP_DIR}/patches
+COPY patches ${APP_DIR}/patches
 
-# RUN for d in $APP_DIR/patches/*; do \
-#     if [ -d $d ]; then \
-#     for f in `ls $d/*.patch | sort -g`; do \
-#     cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
-#     done ; \
-#     fi ; \
-#     done
+RUN for d in $APP_DIR/patches/*; do \
+    if [ -d $d ]; then \
+    for f in `ls $d/*.patch | sort -g`; do \
+    cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+    done ; \
+    fi ; \
+    done

ckan/docker-entrypoint.d/02_setup_scheming.sh

@@ -1,16 +1,18 @@
 #!/bin/bash
 
-# Update ckanext-scheming and ckanext-facet_scheming settings defined in the env var
-echo "Set up ckanext-facet_scheming. Clear index"
+# Update ckanext-scheming and ckanext-scheming_dcat settings defined in the env var
+echo "Set up ckanext-scheming_dcat. Clear index"
 ckan -c $CKAN_INI search-index clear
 
-echo "Loading ckanext-scheming and ckanext-facet_scheming settings into ckan.ini"
+echo "Loading ckanext-scheming and ckanext-scheming_dcat settings into ckan.ini"
 ckan config-tool $CKAN_INI \
-    "scheming.dataset_schemas=$SCHEMA_CKANEXT_SCHEMING_DATASET_SCHEMA" \
-    "scheming.group_schemas=$SCHEMA_CKANEXT_SCHEMING_GROUP_SCHEMAS" \
-    "scheming.organization_schemas=$SCHEMA_CKANEXT_SCHEMING_ORGANIZATION_SCHEMAS" \
-    "scheming.presets=$SCHEMA_CKANEXT_SCHEMING_PRESETS" \
-    "facet_scheming.facet_list=$CKANEXT__FACET_FACET_LIST"
+    "scheming.dataset_schemas=$CKANEXT__SCHEMING_DCAT_DATASET_SCHEMA" \
+    "scheming.group_schemas=$CKANEXT__SCHEMING_DCAT_GROUP_SCHEMAS" \
+    "scheming.organization_schemas=$CKANEXT__SCHEMING_DCAT_ORGANIZATION_SCHEMAS" \
+    "scheming.presets=$CKANEXT__SCHEMING_DCAT_PRESETS" \
+    "scheming_dcat.facet_list=$CKANEXT__SCHEMING_DCAT_FACET_LIST" \
+    "scheming_dcat.organization_custom_facets=$CKANEXT__SCHEMING_DCAT_ORGANIZATION_CUSTOM_FACETS" \
+    "scheming_dcat.group_custom_facets=$CKANEXT__SCHEMING_DCAT_GROUP_CUSTOM_FACETS"
 
-echo "ckanext-facet_scheming. Rebuild index"
+echo "ckanext-scheming_dcat. Rebuild index"
 ckan -c $CKAN_INI search-index rebuild

doc/scripts/ckan_backup-zip.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Set the necessary variables
+CONTAINER_NAME="db"
+DATABASE_NAME="ckan"
+POSTGRES_USER="ckan"
+POSTGRES_PASSWORD="your_postgres_password"
+BACKUP_DIRECTORY="/path/to/your/backup/directory"
+DATE=$(date +%Y%m%d%H%M%S)
+MONTH=$(date +%m)
+YEAR=$(date +%Y)
+
+# Create the monthly backup directory if it doesn't exist
+mkdir -p "$BACKUP_DIRECTORY/monthly/$YEAR-$MONTH"
+
+# Run the backup command
+docker exec -e PGPASSWORD=$POSTGRES_PASSWORD $CONTAINER_NAME pg_dump -U $POSTGRES_USER -Fc $DATABASE_NAME > "$BACKUP_DIRECTORY/monthly/$YEAR-$MONTH/ckan_backup_$DATE.dump"
+
+# Compress the dump files into a zip archive
+cd "$BACKUP_DIRECTORY/monthly/$YEAR-$MONTH" || exit
+zip "backup_${YEAR}-${MONTH}.zip" *.dump
+
+# Remove the original dump files
+rm -f *.dump

doc/scripts/ckan_backup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Set the necessary variables
+CONTAINER_NAME="db"
+DATABASE_NAME="ckan"
+POSTGRES_USER="ckan"
+POSTGRES_PASSWORD="your_postgres_password"
+BACKUP_DIRECTORY="/path/to/your/backup/directory"
+DATE=$(date +%Y%m%d%H%M%S)
+
+# Run the backup command
+docker exec -e PGPASSWORD=$POSTGRES_PASSWORD $CONTAINER_NAME pg_dump -U $POSTGRES_USER -Fc $DATABASE_NAME > $BACKUP_DIRECTORY/ckan_backup_$DATE.dump

nginx/Dockerfile

@@ -18,7 +18,7 @@ RUN mkdir -p ${NGINX_LOG_DIR}
 
 COPY setup/nginx.conf ${NGINX_DIR}/nginx.conf
 COPY setup/index.html /usr/share/nginx/html/index.html
-COPY setup/default.conf.template ${NGINX_DIR}/templates/default.conf.templateç
+COPY setup/default.conf.template ${NGINX_DIR}/templates/default.conf.template
 
 RUN mkdir -p ${NGINX_DIR}/certs
 COPY setup/ckan-local.* ${NGINX_DIR}/certs/

samples/.env.apache.example

@@ -101,10 +101,10 @@ CKANEXT__DCAT__BASE_URI=https://localhost:5000/
 CKANEXT__DCAT__RDF_PROFILES="euro_dcat_ap_2 euro_dcat_ap"
 
 # ckanext-scheming: setup_scheming.sh
-SCHEMA_CKANEXT_SCHEMING_DATASET_SCHEMA="ckanext.scheming:ckan_geodcatap.yaml"
-SCHEMA_CKANEXT_SCHEMING_GROUP_SCHEMAS="ckanext.scheming:ckan_group_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_ORGANIZATION_SCHEMAS="ckanext.scheming:ckan_org_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_PRESETS="ckanext.scheming:presets.json"
+CKANEXT__SCHEMING_DCAT_DATASET_SCHEMA="ckanext.scheming:ckan_geodcatap.yaml"
+CKANEXT__SCHEMING_DCAT_GROUP_SCHEMAS="ckanext.scheming:ckan_group_geodcatap.json"
+CKANEXT__SCHEMING_DCAT_ORGANIZATION_SCHEMAS="ckanext.scheming:ckan_org_geodcatap.json"
+CKANEXT__SCHEMING_DCAT_PRESETS="ckanext.scheming:presets.json"
 
 # ckanext-spatial (Solr Backend - solr8-spatial)
 CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox

samples/.env.localhost

@@ -123,10 +123,10 @@ CKANEXT__DCAT__BASE_URI=http://localhost/catalog
 CKANEXT__DCAT__RDF_PROFILES="euro_dcat_ap_2 euro_dcat_ap"
 
 # ckanext-scheming: setup_scheming.sh
-SCHEMA_CKANEXT_SCHEMING_DATASET_SCHEMA="ckanext.scheming:ckan_geodcatap.yaml"
-SCHEMA_CKANEXT_SCHEMING_GROUP_SCHEMAS="ckanext.scheming:ckan_group_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_ORGANIZATION_SCHEMAS="ckanext.scheming:ckan_org_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_PRESETS="ckanext.scheming:presets.json"
+CKANEXT__SCHEMING_DCAT_DATASET_SCHEMA="ckanext.scheming:ckan_geodcatap.yaml"
+CKANEXT__SCHEMING_DCAT_GROUP_SCHEMAS="ckanext.scheming:ckan_group_geodcatap.json"
+CKANEXT__SCHEMING_DCAT_ORGANIZATION_SCHEMAS="ckanext.scheming:ckan_org_geodcatap.json"
+CKANEXT__SCHEMING_DCAT_PRESETS="ckanext.scheming:presets.json"
 
 # ckanext-spatial (Solr Backend - solr8-spatial)
 CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox

samples/.env.nginx.example

@@ -101,10 +101,10 @@ CKANEXT__DCAT__BASE_URI=https://localhost:5000/
 CKANEXT__DCAT__RDF_PROFILES="euro_dcat_ap_2 euro_dcat_ap"
 
 # ckanext-scheming: setup_scheming.sh
-SCHEMA_CKANEXT_SCHEMING_DATASET_SCHEMA="ckanext.scheming:ckan_geodcatap.yaml"
-SCHEMA_CKANEXT_SCHEMING_GROUP_SCHEMAS="ckanext.scheming:ckan_group_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_ORGANIZATION_SCHEMAS="ckanext.scheming:ckan_org_geodcatap.json"
-SCHEMA_CKANEXT_SCHEMING_PRESETS="ckanext.scheming:presets.json"
+CKANEXT__SCHEMING_DCAT_DATASET_SCHEMA="ckanext.scheming:ckan_geodcatap.yaml"
+CKANEXT__SCHEMING_DCAT_GROUP_SCHEMAS="ckanext.scheming:ckan_group_geodcatap.json"
+CKANEXT__SCHEMING_DCAT_ORGANIZATION_SCHEMAS="ckanext.scheming:ckan_org_geodcatap.json"
+CKANEXT__SCHEMING_DCAT_PRESETS="ckanext.scheming:presets.json"
 
 # ckanext-spatial (Solr Backend - solr8-spatial)
 CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox