From 0f877d55646140a4cea5886c3c69fb2d2f66e349 Mon Sep 17 00:00:00 2001 From: Pavle Jonoski Date: Tue, 5 Mar 2024 15:44:40 +0100 Subject: [PATCH 1/4] Move over docker related files. --- .env | 54 +++ docker-compose.yml | 117 ++++++ docker/ckan/Dockerfile | 86 +++++ docker/ckan/development.ini | 352 ++++++++++++++++++ docker/ckan/setup/start_ckan_development.sh | 91 +++++ docker/ckan/setup/start_ckan_worker.sh | 56 +++ docker/postgresql/Dockerfile | 14 + .../10_create_datastore.sql | 4 + .../20_setup_test_databases.sql | 2 + docker/solr/Dockerfile | 32 ++ docker/solr/solrconfig.xml | 343 +++++++++++++++++ 11 files changed, 1151 insertions(+) create mode 100644 .env create mode 100644 docker-compose.yml create mode 100644 docker/ckan/Dockerfile create mode 100644 docker/ckan/development.ini create mode 100755 docker/ckan/setup/start_ckan_development.sh create mode 100755 docker/ckan/setup/start_ckan_worker.sh create mode 100644 docker/postgresql/Dockerfile create mode 100644 docker/postgresql/docker-entrypoint-initdb.d/10_create_datastore.sql create mode 100644 docker/postgresql/docker-entrypoint-initdb.d/20_setup_test_databases.sql create mode 100644 docker/solr/Dockerfile create mode 100644 docker/solr/solrconfig.xml diff --git a/.env b/.env new file mode 100644 index 0000000..91bdb39 --- /dev/null +++ b/.env @@ -0,0 +1,54 @@ +# DB image settings +POSTGRES_PASSWORD=ckan +DATASTORE_READONLY_PASSWORD=datastore + +# Basic +CKAN_SITE_ID=default +CKAN_SITE_URL=http://ckan.iaea.local:5000 +CKAN_PORT=5000 +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +TZ=UTC + +# Database connections (TODO: avoid duplication) +CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore + +# Test database connections +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test + +# Other services connections +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_DATAPUSHER_URL=http://datapusher:8800 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 + +TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan +TEST_CKAN_REDIS_URL=redis://redis:6379/1 + +# Core settings +CKAN__STORAGE_PATH=/var/lib/ckan + +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost + +## Extensions +CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore datapusher iaea validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations +CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view geojson_view +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=6379 +CKAN__HARVEST__MQ__REDIS_DB=1 + +# Sentry +# CKAN___SENTRY__DSN=https://xxxxxx:xxxxxx@sentry.domain.com/1 +# CKAN___SENTRY__CONFIGURE_LOGGING=True +# CKAN___SENTRY__LOG_LEVEL=WARN + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e4e8a60 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,117 @@ +version: "3" + +services: + ckan: + profiles: ["full"] + hostname: ckan.iaea.local + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + env_file: + - .env + links: + - db + - solr + - datapusher + - redis + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + volumes: + - .:/srv/app/src_extensions/ckanext-iaea + - ckan_storage:/var/lib/ckan + + ckan-worker-default: + profiles: ["full"] + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + env_file: + - .env + links: + - db + - solr + - datapusher + - redis + volumes: + - .:/srv/app/src_extensions/ckanext-iaea + - ckan_storage:/var/lib/ckan + command: ["/srv/app/start_ckan_worker.sh", "default"] + + ckan-worker-bulk: + profiles: ["full"] + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + env_file: + - .env + links: + - db + - solr + - datapusher + - redis + volumes: + - .:/srv/app/src_extensions/ckanext-iaea + - ckan_storage:/var/lib/ckan + command: ["/srv/app/start_ckan_worker.sh", "bulk"] + + ckan-worker-priority: + profiles: ["full"] + build: + context: ckan/ + dockerfile: Dockerfile + args: + - TZ=${TZ} + env_file: + - .env + links: + - db + - solr + - datapusher + - redis + volumes: + - .:/srv/app/src_extensions/ckanext-iaea + - ckan_storage:/var/lib/ckan + command: ["/srv/app/start_ckan_worker.sh", "priority"] + + db: + container_name: db + env_file: + - .env + environment: + - "PGDATA=/var/lib/postgresql/data/dbdata" + ports: + - "5432:5432" + build: + context: postgresql/ + volumes: + - pg_data:/var/lib/postgresql/data/dbdata + + solr: + container_name: solr + ulimits: + nofile: + soft: 65536 + hard: 65536 + build: + context: solr/ + ports: + - "8983:8983" + volumes: + - solr_data:/opt/solr/server/solr/ckan/data/index + + redis: + container_name: redis + image: redis:alpine + ports: + - "6379:6379" + +volumes: + ckan_storage: + pg_data: + solr_data: \ No newline at end of file diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile new file mode 100644 index 0000000..c47dd7a --- /dev/null +++ b/docker/ckan/Dockerfile @@ -0,0 +1,86 @@ +FROM ckan/ckan-base:2.9 + +MAINTAINER Open Knowledge International + +ENV APP_DIR=/srv/app +ENV SRC_EXTENSIONS_DIR=/srv/app/src_extensions +ENV IAEA_EXTENSION_DIR=/srv/app/src_extensions/ckanext-iaea + +# Set timezone +ARG TZ +RUN cp /usr/share/zoneinfo/$TZ /etc/localtime +RUN echo $TZ > /etc/timezone + +# Install packages needed by the dev requirements +RUN apk add --no-cache libffi-dev + +# Install CKAN dev requirements +#RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt +RUN pip install flask-debugtoolbar + + +# Install CKAN extensions +RUN pip install -e "git+https://github.com/ckan/ckanext-pdfview.git@0.0.7#egg=ckanext-pdfview" && \ + pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages + +# Install ckanext-harvest +RUN pip install -e git+https://github.com/ckan/ckanext-harvest.git#egg=ckanext-harvest +RUN pip install -r ./src/ckanext-harvest/requirements.txt + +# ckanext-autz-service +RUN pip install -e git+https://github.com/datopian/ckanext-authz-service.git#egg=ckanext-authz-service && \ + pip install -r ./src/ckanext-authz-service/requirements.py2.txt + +# ckanext-sentry +RUN pip install -e git+https://github.com/okfn/ckanext-sentry.git#egg=ckanext-sentry && \ + pip install -r ./src/ckanext-sentry/requirements.txt + +# ckanext-basiccharts +RUN pip install -e git+https://github.com/ckan/ckanext-basiccharts.git#egg=ckanext-basiccharts + +# ckanext-visualize +RUN pip install -e git+https://github.com/datopian/ckanext-visualize.git#egg=ckanext-visualize && \ + pip install -r ./src/ckanext-visualize/requirements.txt + +# ckanext-geoview +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-geoview.git@iaea#egg=ckanext-geoview" + +RUN pip install -e "git+https://github.com/ckan/ckanext-pages.git@v0.3.7#egg=ckanext-pages" && \ + pip install -r ./src/ckanext-pages/requirements.txt + +# ckanext-dataexplorer-react +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-dataexplorer-react.git@iaea#egg=ckanext-dataexplorer-react" && \ + pip install -r ./src/ckanext-dataexplorer-react/requirements.txt + +# ckanext-dcat +RUN pip install -e "git+https://github.com/ckan/ckanext-dcat.git@v1.3.0#egg=ckanext-dcat" && \ + pip install -r ./src/ckanext-dcat/requirements-py2.txt + +# ckanext-report +RUN pip install -e "git+https://github.com/ckan/ckanext-report.git#egg=ckanext-report" && \ + pip install -r ./src/ckanext-report/requirements.txt + +# ckanext-archiver +RUN pip install -e "git+http://github.com/ckan/ckanext-archiver.git#egg=ckanext-archiver" && \ + pip install -r ./src/ckanext-archiver/requirements.txt + +# ckanext-qa +RUN pip install -e "git+http://github.com/keitaroinc/ckanext-qa.git@qa-iaea#egg=ckanext-qa" && \ + pip install -r ./src/ckanext-qa/requirements.txt + +# ckanext-validation +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-validation.git@ckan-2.8.2#egg=ckanext-validation" && \ + pip install -r ./src/ckanext-validation/requirements.txt + +# ckanext-scheming +RUN pip install --exists-action=w -e "git+https://github.com/keitaroinc/ckanext-scheming.git@ckan-2.8#egg=ckanext-scheming" + +# Create folder for local extensions sources +RUN mkdir $SRC_EXTENSIONS_DIR +RUN mkdir $IAEA_EXTENSION_DIR + +COPY setup/start_ckan_development.sh ${APP_DIR} +COPY setup/start_ckan_worker.sh ${APP_DIR} + + +CMD ["/srv/app/start_ckan_development.sh"] diff --git a/docker/ckan/development.ini b/docker/ckan/development.ini new file mode 100644 index 0000000..a97267d --- /dev/null +++ b/docker/ckan/development.ini @@ -0,0 +1,352 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* +debug = false + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:ckan +full_stack = true +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# Enable tracking +ckan.tracking_enabled = true + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `paster make-config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = p3yVBxMItfwQtsXOE++gZatJO + +# `paster make-config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = 08bfdc21-977a-4a83-ae15-4e8cfbc0120a + +# repoze.who config +who.config_file = /usr/lib/ckan/default/src/ckan/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +# who.timeout = 86400 + +## Database Settings +sqlalchemy.url = postgresql://ckan:ckan@localhost:5432/ckan + +ckan.datastore.write_url = postgresql://ckan:ckan@localhost:5432/datastore +ckan.datastore.read_url = postgresql://datastore_ro:datastore@localhost:5432/datastore + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist +ckan.datastore.sqlsearch.allowed_functions_file=/srv/app/allowed_functions.txt + + +## Site Settings + +ckan.site_url = http://ckan.iaea.local:5000 +#ckan.site_url = https://data-dev.iaea.org +#ckan.use_pylons_response_cleanup_middleware = true + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = true +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true +ckan.auth.roles_that_cascade_to_sub_groups = admin + + +## Search Settings + +ckan.site_id = default +solr_url = http://127.0.0.1:8983/solr/ckan + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +ckan.redis.url = redis://localhost:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +#ckan.plugins = envvars image_view text_view recline_view webpage_view pdf_view pages datastore datapusher harvest ckan_harvester iaea iaea_db_harvester authz_service +#ckan.plugins = envvars image_view text_view recline_view +#ckan.plugins = envvars stats text_view recline_view image_view webpage_view resource_proxy datastore datapusher report archiver qa harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data iaea +#ckan.plugins = envvars stats text_view image_view webpage_view resource_proxy datastore xloader visualize iaea validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations saml2auth +ckan.plugins = envvars stats text_view image_view webpage_view resource_proxy datastore xloader validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations iaea saml2auth + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +#ckan.views.default_views = image_view text_view recline_view +ckan.views.default_views = image_view text_view recline_view pdf_view video_view geojson_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +# Uncomment following configuration to enable using of Bootstrap 2 +#ckan.base_public_folder = public-bs2 +#ckan.base_templates_folder = templates-bs2 + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +ckan.storage_path = /usr/lib/ckan/iaea-storage +ckan.max_resource_size = 100 +#ckan.max_image_size = 2 + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = + + +## Other extensions + +# Sentry +#sentry.dsn = https://xxxxxx:xxxxxx@sentry.domain.com/1 +#sentry.configure_logging=True +#sentry.log_level=WARN + +# Autz-Service +ckanext.authz_service.jwt_private_key_file=./test-keys/jwt-rs256.key +ckanext.authz_service.jwt_public_key_file=./test-keys/jwt-rs256.key.pub +ckanext.authz_service.jwt_algorithm=RS256 + + +# Googleanalytics +ckanext.iaea.googleanalytics.id=G-REPLACEME + +# Sets default ogranization for new datasets +ckanext.iaea.main_organization=iaea +ckanext.iaea.allow_dataset_create_from_organization=iaea,load_test + +## ckanext-saml2auth +# Specifies the metadata location type +# Options: local or remote +ckanext.saml2auth.idp_metadata.location = local + +# Path to a local file accessible on the server the service runs on +# Ignore this config if the idp metadata location is set to: remote +ckanext.saml2auth.idp_metadata.local_path = sso/sso_idp.xml + +# A remote URL serving aggregate metadata +# Ignore this config if the idp metadata location is set to: local +# ckanext.saml2auth.idp_metadata.remote_url = https://kalmar2.org/simplesaml/module.php/aggregator/?id=kalmarcentral2&set=saml2 + +# Path to a local file accessible on the server the service runs on +# Ignore this config if the idp metadata location is set to: local and metadata is public +# ckanext.saml2auth.idp_metadata.remote_cert = /opt/metadata/kalmar2.cert + +# Corresponding SAML user field for firstname +ckanext.saml2auth.user_firstname = FirstName + +# Corresponding SAML user field for lastname +ckanext.saml2auth.user_lastname = LastName + +# Corresponding SAML user field for fullname +# (Optional: Can be used as an alternative to firstname + lastname) +# ckanext.saml2auth.user_fullname = fullname + +# Corresponding SAML user field for email +ckanext.saml2auth.user_email = EmailAddress + +# URL route of the endpoint where the SAML assertion is sent, also known as Assertion Consumer Service (ACS). +# Default: /acs +ckanext.saml2auth.acs_endpoint = /sso/post + +# Configuration setting that enables CKAN's internal register/login functionality as well +# Default: False +ckanext.saml2auth.enable_ckan_internal_login = True + +ckanext.saml2auth.want_assertions_or_response_signed = False +ckanext.saml2auth.want_response_signed = False +ckanext.saml2auth.key_file_path=sso/iaea-dev.key +ckanext.saml2auth.cert_file_path=sso/iaea-dev.crt +ckanext.saml2auth.sp.name_id_format=urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified + +# Entity ID (also know as Issuer) +# Define the entity ID. Default is urn:mace:umu.se:saml:ckan:sp +ckanext.saml2auth.entity_id=https://data-dev.iaea.org + + +ckanext.saml2auth.user_default_org=orgone +ckanext.saml2auth.user_default_role=admin + +#ckanext.saml2auth.sysadmins_list= + +## CKAN Scheming +scheming.dataset_schemas = ckanext.iaea:iaea_dataset_schema.yaml +scheming.presets = ckanext.scheming:presets.json ckanext.iaea:presets.json ckanext.validation:presets.json + +## Xloader +ckanext.xloader.jobs_db.uri = sqlite:////tmp/xloader_jobs.db +ckanext.xloader.max_content_length = 5000000001 +ckanext.xloader.job_timeout=36000 +ckanext.xloader.max_excerpt_lines=0 +ckanext.xloader.ssl_verify=false +ckanext.xloader.formats=csv application/csv xls application/vnd.ms-excel xlsx application/vnd.oasis.opendocument.spreadsheet +ckanext.xloader.rewrite_site_url=http://ckan.iaea.local:5000 + +# Validation +ckanext.validation.formats=csv xlsx xls +ckanext.validation.run_on_create_async=True +ckanext.validation.run_on_update_async=True +ckanext.validation.run_on_create_sync=False +ckanext.validation.run_on_update_sync=False +ckanext.validation.show_badges_in_listings=True + +# Harvest +ckan.harvest.mq.type=redis +ckan.harvest.mq.hostname=localhost +ckan.harvest.mq.port=6379 +ckan.harvest.mq.db=0 + +# Pages +ckanext.pages.editor = ckeditor + +# Dcat +ckanext.dcat.enable_content_negotiation=True + + +## Logging configuration +[loggers] +keys = root, ckan, ckanext + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[logger_saml2] +level = DEBUG +handlers = console +qualname = saml2 +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s \ No newline at end of file diff --git a/docker/ckan/setup/start_ckan_development.sh b/docker/ckan/setup/start_ckan_development.sh new file mode 100755 index 0000000..736ab09 --- /dev/null +++ b/docker/ckan/setup/start_ckan_development.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Install any local extensions in the src_extensions volume +echo "Looking for local extensions to install..." +echo "Extension dir contents:" +ls -la $SRC_EXTENSIONS_DIR +for i in $SRC_EXTENSIONS_DIR/* +do + if [ -d $i ]; + then + + if [ -f $i/pip-requirements.txt ]; + then + pip install -r $i/pip-requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/requirements.txt ]; + then + pip install -r $i/requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/dev-requirements.txt ]; + then + pip install -r $i/dev-requirements.txt + echo "Found dev-requirements file in $i" + fi + if [ -f $i/setup.py ]; + then + cd $i + python $i/setup.py develop + echo "Found setup.py file in $i" + cd $APP_DIR + fi + + # Point `use` in test.ini to location of `test-core.ini` + if [ -f $i/test.ini ]; + then + echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" + paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + fi + fi +done + +# Set debug to true +echo "Enabling debug mode" +paster --plugin=ckan config-tool $CKAN_INI -s DEFAULT "debug = true" + +# Update the plugins setting in the ini file with the values defined in the env var +echo "Loading the following plugins: $CKAN__PLUGINS" +paster --plugin=ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" +paster --plugin=ckan config-tool $CKAN_INI "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" + +paster --plugin=ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" + +# Update test-core.ini DB, SOLR & Redis settings +echo "Loading test settings into test-core.ini" +paster --plugin=ckan config-tool $SRC_DIR/ckan/test-core.ini \ + "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ + "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ + "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ + "solr_url = $TEST_CKAN_SOLR_URL" \ + "ckan.redis.url = $TEST_CKAN_REDIS_URL" + +# Run the prerun script to init CKAN and create the default admin user +sudo -u ckan -EH python prerun.py + +echo "Running DB init scripts" +paster --plugin=ckanext-archiver archiver init --config="$CKAN_INI" +paster --plugin=ckanext-report report initdb --config="$CKAN_INI" +paster --plugin=ckanext-qa qa init --config="$CKAN_INI" +paster --plugin=ckanext-validation validation init-db --config="$CKAN_INI" +echo "Init DB scripts completed." + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Start supervisord +# supervisord --configuration /etc/supervisord.conf & + +# Start the development server with automatic reload +sudo -u ckan -EH paster serve --reload $CKAN_INI diff --git a/docker/ckan/setup/start_ckan_worker.sh b/docker/ckan/setup/start_ckan_worker.sh new file mode 100755 index 0000000..5819e41 --- /dev/null +++ b/docker/ckan/setup/start_ckan_worker.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Install any local extensions in the src_extensions volume +echo "Looking for local extensions to install..." +echo "Extension dir contents:" +ls -la $SRC_EXTENSIONS_DIR +for i in $SRC_EXTENSIONS_DIR/* +do + if [ -d $i ]; + then + + if [ -f $i/pip-requirements.txt ]; + then + pip install -r $i/pip-requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/requirements.txt ]; + then + pip install -r $i/requirements.txt + echo "Found requirements file in $i" + fi + if [ -f $i/dev-requirements.txt ]; + then + pip install -r $i/dev-requirements.txt + echo "Found dev-requirements file in $i" + fi + if [ -f $i/setup.py ]; + then + cd $i + python $i/setup.py develop + echo "Found setup.py file in $i" + cd $APP_DIR + fi + + # Point `use` in test.ini to location of `test-core.ini` + if [ -f $i/test.ini ]; + then + echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" + paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + fi + fi +done + + +# Set debug to true +echo "Enabling debug mode" +paster --plugin=ckan config-tool $CKAN_INI -s DEFAULT "debug = true" + +# Update the plugins setting in the ini file with the values defined in the env var +echo "Loading the following plugins: $CKAN__PLUGINS" +paster --plugin=ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" + +paster --plugin=ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" + +echo "Running worker: $1" +paster --plugin=ckan jobs worker "$1" -c "$CKAN_INI" \ No newline at end of file diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile new file mode 100644 index 0000000..f221611 --- /dev/null +++ b/docker/postgresql/Dockerfile @@ -0,0 +1,14 @@ +FROM postgres:9.6-alpine +MAINTAINER Open Knowledge International + +# Allow connections; we don't map out any ports so only linked docker containers can connect +RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf + +# Customize default user/pass/db +ENV POSTGRES_DB ckan +ENV POSTGRES_USER ckan +ARG POSTGRES_PASSWORD +ARG DATASTORE_READONLY_PASSWORD + +# Include extra setup scripts (eg datastore) +ADD docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/docker/postgresql/docker-entrypoint-initdb.d/10_create_datastore.sql b/docker/postgresql/docker-entrypoint-initdb.d/10_create_datastore.sql new file mode 100644 index 0000000..a811507 --- /dev/null +++ b/docker/postgresql/docker-entrypoint-initdb.d/10_create_datastore.sql @@ -0,0 +1,4 @@ +\set datastore_ro_password '\'' `echo $DATASTORE_READONLY_PASSWORD` '\'' + +CREATE ROLE datastore_ro NOSUPERUSER LOGIN PASSWORD :datastore_ro_password; +CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; diff --git a/docker/postgresql/docker-entrypoint-initdb.d/20_setup_test_databases.sql b/docker/postgresql/docker-entrypoint-initdb.d/20_setup_test_databases.sql new file mode 100644 index 0000000..140f2e5 --- /dev/null +++ b/docker/postgresql/docker-entrypoint-initdb.d/20_setup_test_databases.sql @@ -0,0 +1,2 @@ +CREATE DATABASE ckan_test OWNER ckan ENCODING 'utf-8'; +CREATE DATABASE datastore_test OWNER ckan ENCODING 'utf-8'; diff --git a/docker/solr/Dockerfile b/docker/solr/Dockerfile new file mode 100644 index 0000000..2484739 --- /dev/null +++ b/docker/solr/Dockerfile @@ -0,0 +1,32 @@ +FROM solr:6.5.1-alpine +MAINTAINER Open Knowledge + +# Enviroment +ENV SOLR_CORE ckan +# Adjust to your needs +ENV CKAN_VERSION dev-v2.8 + +# User +USER root + +# Create Directories +RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf +RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data + +# Adding Files +ADD solrconfig.xml \ +https://raw.githubusercontent.com/ckan/ckan/$CKAN_VERSION/ckan/config/solr/schema.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +/opt/solr/server/solr/$SOLR_CORE/conf/ + +# Create Core.properties +RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties + +# Giving ownership to Solr +RUN chown -R solr:solr /opt/solr/server/solr/$SOLR_CORE + +USER solr diff --git a/docker/solr/solrconfig.xml b/docker/solr/solrconfig.xml new file mode 100644 index 0000000..9ac620c --- /dev/null +++ b/docker/solr/solrconfig.xml @@ -0,0 +1,343 @@ + + + + + + 6.0.0 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + ${solr.lock.type:native} + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + ${solr.autoCommit.maxTime:15000} + false + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + 1024 + + + + + true + 20 + 200 + + + + + + + + + + false + 2 + + + + + + + + + + + + + + explicit + 10 + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + + + _text_ + + + + + + + + add-unknown-fields-to-the-schema + + + + + + + + true + ignored_ + _text_ + + + + + + + + + + + explicit + true + + + + + + + text_general + + + default + _text_ + solr.DirectSolrSpellChecker + internal + 0.5 + 2 + 1 + 5 + 4 + 0.01 + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + true + false + + + + terms + + + + + + + string + elevate.xml + + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + 70 + 0.5 + [-\w ,/\n\"']{20,200} + + + + + + ]]> + ]]> + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + + 10 + .,!? + + + + + + WORD + en + US + + + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + text/plain; charset=UTF-8 + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + 5 + + + From 80d15de47a279e3f67075486aee980bee44427ff Mon Sep 17 00:00:00 2001 From: Pavle Jonoski Date: Thu, 7 Mar 2024 00:06:59 +0100 Subject: [PATCH 2/4] Updates docker compose script and Dockerfile for CKAN 2.9. --- .env | 32 +- docker-compose.yml | 122 ++++--- docker/ckan/Dockerfile | 74 ++-- docker/ckan/ckan.ini | 258 ++++++++++++++ docker/ckan/development.ini | 352 -------------------- docker/ckan/setup/start_ckan_development.sh | 28 +- docker/ckan/setup/start_ckan_worker.sh | 12 +- docker/postgresql/Dockerfile | 2 +- docker/solr/Dockerfile | 32 -- docker/solr/solrconfig.xml | 343 ------------------- 10 files changed, 384 insertions(+), 871 deletions(-) create mode 100644 docker/ckan/ckan.ini delete mode 100644 docker/ckan/development.ini delete mode 100644 docker/solr/Dockerfile delete mode 100644 docker/solr/solrconfig.xml diff --git a/.env b/.env index 91bdb39..bada14c 100644 --- a/.env +++ b/.env @@ -12,23 +12,23 @@ CKAN_SYSADMIN_EMAIL=your_email@example.com TZ=UTC # Database connections (TODO: avoid duplication) -CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@db/ckan -CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore -CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore +CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@iaea_db/ckan +CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@iaea_db/datastore +CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@iaea_db/datastore # Test database connections -TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test -TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test -TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test +TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@iaea_db/ckan_test +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@iaea_db/datastore_test +TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@iaea_db/datastore_test # Other services connections -CKAN_SOLR_URL=http://solr:8983/solr/ckan -CKAN_REDIS_URL=redis://redis:6379/1 -CKAN_DATAPUSHER_URL=http://datapusher:8800 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 +CKAN_SOLR_URL=http://iaea_solr:8983/solr/ckan +CKAN_REDIS_URL=redis://iaea_redis:6379/1 +#CKAN_DATAPUSHER_URL=http://datapusher:8800 +#CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 -TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan -TEST_CKAN_REDIS_URL=redis://redis:6379/1 +TEST_CKAN_SOLR_URL=http://iaea_solr:8983/solr/ckan +TEST_CKAN_REDIS_URL=redis://iaea_redis:6379/1 # Core settings CKAN__STORAGE_PATH=/var/lib/ckan @@ -40,10 +40,12 @@ CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost ## Extensions -CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore datapusher iaea validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations -CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view geojson_view +#CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore datapusher iaea validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations +CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore +#CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view geojson_view +CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__HOSTNAME=iaea_redis CKAN__HARVEST__MQ__PORT=6379 CKAN__HARVEST__MQ__REDIS_DB=1 diff --git a/docker-compose.yml b/docker-compose.yml index e4e8a60..35e767c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,8 @@ version: "3" services: ckan: - profiles: ["full"] - hostname: ckan.iaea.local build: - context: ckan/ + context: docker/ckan/ dockerfile: Dockerfile args: - TZ=${TZ} @@ -14,73 +12,66 @@ services: links: - db - solr - - datapusher - redis ports: - - "0.0.0.0:${CKAN_PORT}:5000" + - "${CKAN_PORT}:5000" volumes: - - .:/srv/app/src_extensions/ckanext-iaea + #- .:/srv/app/src_extensions/ckanext-iaea - ckan_storage:/var/lib/ckan - ckan-worker-default: - profiles: ["full"] - build: - context: ckan/ - dockerfile: Dockerfile - args: - - TZ=${TZ} - env_file: - - .env - links: - - db - - solr - - datapusher - - redis - volumes: - - .:/srv/app/src_extensions/ckanext-iaea - - ckan_storage:/var/lib/ckan - command: ["/srv/app/start_ckan_worker.sh", "default"] + # ckan-worker-default: + # build: + # context: docker/ckan/ + # dockerfile: Dockerfile + # args: + # - TZ=${TZ} + # env_file: + # - .env + # links: + # - db + # - solr + # - redis + # volumes: + # - .:/srv/app/src_extensions/ckanext-iaea + # - ckan_storage:/var/lib/ckan + # command: ["/srv/app/start_ckan_worker.sh", "default"] - ckan-worker-bulk: - profiles: ["full"] - build: - context: ckan/ - dockerfile: Dockerfile - args: - - TZ=${TZ} - env_file: - - .env - links: - - db - - solr - - datapusher - - redis - volumes: - - .:/srv/app/src_extensions/ckanext-iaea - - ckan_storage:/var/lib/ckan - command: ["/srv/app/start_ckan_worker.sh", "bulk"] + # ckan-worker-bulk: + # build: + # context: docker/ckan/ + # dockerfile: Dockerfile + # args: + # - TZ=${TZ} + # env_file: + # - .env + # links: + # - db + # - solr + # - redis + # volumes: + # - .:/srv/app/src_extensions/ckanext-iaea + # - ckan_storage:/var/lib/ckan + # command: ["/srv/app/start_ckan_worker.sh", "bulk"] - ckan-worker-priority: - profiles: ["full"] - build: - context: ckan/ - dockerfile: Dockerfile - args: - - TZ=${TZ} - env_file: - - .env - links: - - db - - solr - - datapusher - - redis - volumes: - - .:/srv/app/src_extensions/ckanext-iaea - - ckan_storage:/var/lib/ckan - command: ["/srv/app/start_ckan_worker.sh", "priority"] + # ckan-worker-priority: + # build: + # context: docker/ckan/ + # dockerfile: Dockerfile + # args: + # - TZ=${TZ} + # env_file: + # - .env + # links: + # - db + # - solr + # - redis + # volumes: + # - .:/srv/app/src_extensions/ckanext-iaea + # - ckan_storage:/var/lib/ckan + # command: ["/srv/app/start_ckan_worker.sh", "priority"] db: - container_name: db + container_name: iaea_db env_file: - .env environment: @@ -88,25 +79,24 @@ services: ports: - "5432:5432" build: - context: postgresql/ + context: docker/postgresql/ volumes: - pg_data:/var/lib/postgresql/data/dbdata solr: - container_name: solr + image: ckan/ckan-solr:master + container_name: iaea_solr ulimits: nofile: soft: 65536 hard: 65536 - build: - context: solr/ ports: - "8983:8983" volumes: - solr_data:/opt/solr/server/solr/ckan/data/index redis: - container_name: redis + container_name: iaea_redis image: redis:alpine ports: - "6379:6379" diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile index c47dd7a..3fa59d4 100644 --- a/docker/ckan/Dockerfile +++ b/docker/ckan/Dockerfile @@ -14,66 +14,54 @@ RUN echo $TZ > /etc/timezone # Install packages needed by the dev requirements RUN apk add --no-cache libffi-dev -# Install CKAN dev requirements -#RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt +# # Install CKAN dev requirements +# #RUN pip install --no-binary :all: -r https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/dev-requirements.txt RUN pip install flask-debugtoolbar # Install CKAN extensions -RUN pip install -e "git+https://github.com/ckan/ckanext-pdfview.git@0.0.7#egg=ckanext-pdfview" && \ - pip install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages -# Install ckanext-harvest -RUN pip install -e git+https://github.com/ckan/ckanext-harvest.git#egg=ckanext-harvest -RUN pip install -r ./src/ckanext-harvest/requirements.txt +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-xloader@rewrite-site-url#egg=ckanext-xloader" && \ + pip install -r "./src/ckanext-xloader/requirements.txt" -# ckanext-autz-service -RUN pip install -e git+https://github.com/datopian/ckanext-authz-service.git#egg=ckanext-authz-service && \ - pip install -r ./src/ckanext-authz-service/requirements.py2.txt +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-validation@iaea-2.9#egg=ckanext-validation" && \ + pip install -r "./src/ckanext-validation/requirements.txt" -# ckanext-sentry -RUN pip install -e git+https://github.com/okfn/ckanext-sentry.git#egg=ckanext-sentry && \ - pip install -r ./src/ckanext-sentry/requirements.txt +RUN pip install -e "git+https://github.com/ckan/ckanext-report#egg=ckanext-report" && \ + pip install -r "./src/ckanext-report/requirements.txt" -# ckanext-basiccharts -RUN pip install -e git+https://github.com/ckan/ckanext-basiccharts.git#egg=ckanext-basiccharts +RUN pip install -e "git+https://github.com/ckan/ckanext-archiver#egg=ckanext-archiver" && \ + sed -i '/ckanext-report/d' "./src/ckanext-archiver/requirements.txt" && \ + pip install -r "./src/ckanext-archiver/requirements.txt" -# ckanext-visualize -RUN pip install -e git+https://github.com/datopian/ckanext-visualize.git#egg=ckanext-visualize && \ - pip install -r ./src/ckanext-visualize/requirements.txt +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-qa@qa-iaea#egg=ckanext-qa" && \ + pip install -r "./src/ckanext-qa/requirements.txt" -# ckanext-geoview -RUN pip install -e "git+https://github.com/keitaroinc/ckanext-geoview.git@iaea#egg=ckanext-geoview" +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-harvest@montreal-fix#egg=ckanext-harvest" && \ + pip install -r "./src/ckanext-harvest/pip-requirements.txt" -RUN pip install -e "git+https://github.com/ckan/ckanext-pages.git@v0.3.7#egg=ckanext-pages" && \ - pip install -r ./src/ckanext-pages/requirements.txt +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-basiccharts@iaea#egg=ckanext-basiccharts" -# ckanext-dataexplorer-react -RUN pip install -e "git+https://github.com/keitaroinc/ckanext-dataexplorer-react.git@iaea#egg=ckanext-dataexplorer-react" && \ - pip install -r ./src/ckanext-dataexplorer-react/requirements.txt +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-visualize#egg=ckanext-visualize" && \ + pip install -r "./src/ckanext-visualize/requirements.txt" -# ckanext-dcat -RUN pip install -e "git+https://github.com/ckan/ckanext-dcat.git@v1.3.0#egg=ckanext-dcat" && \ - pip install -r ./src/ckanext-dcat/requirements-py2.txt +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-pdfview#egg=ckanext-pdfview" -# ckanext-report -RUN pip install -e "git+https://github.com/ckan/ckanext-report.git#egg=ckanext-report" && \ - pip install -r ./src/ckanext-report/requirements.txt +RUN pip install -e "git+https://github.com/ckan/ckanext-geoview@v0.1.0#egg=ckanext-geoview" -# ckanext-archiver -RUN pip install -e "git+http://github.com/ckan/ckanext-archiver.git#egg=ckanext-archiver" && \ - pip install -r ./src/ckanext-archiver/requirements.txt +RUN pip install -e "git+https://github.com/ckan/ckanext-pages@v0.5.2#egg=ckanext-pages" && \ + pip install -r "./src/ckanext-pages/requirements.txt" -# ckanext-qa -RUN pip install -e "git+http://github.com/keitaroinc/ckanext-qa.git@qa-iaea#egg=ckanext-qa" && \ - pip install -r ./src/ckanext-qa/requirements.txt +RUN pip install -e "git+https://github.com/datopian/ckanext-dataexplorer-react#egg=ckanext-dataexplorer-react" && \ + pip install -r "./src/ckanext-dataexplorer-react/requirements.txt" -# ckanext-validation -RUN pip install -e "git+https://github.com/keitaroinc/ckanext-validation.git@ckan-2.8.2#egg=ckanext-validation" && \ - pip install -r ./src/ckanext-validation/requirements.txt +RUN pip install -e "git+https://github.com/ckan/ckanext-dcat@v1.5.1#egg=ckanext-dcat" && \ + pip install -r "./src/ckanext-dcat/requirements.txt" + +RUN pip install -e "git+https://github.com/ckan/ckanext-scheming@release-3.0.0#egg=ckanext-scheming" + +RUN pip install -e "git+https://github.com/keitaroinc/ckanext-saml2auth@ckan-2.9-iaea#egg=ckanext-saml2auth" -# ckanext-scheming -RUN pip install --exists-action=w -e "git+https://github.com/keitaroinc/ckanext-scheming.git@ckan-2.8#egg=ckanext-scheming" # Create folder for local extensions sources RUN mkdir $SRC_EXTENSIONS_DIR @@ -81,6 +69,8 @@ RUN mkdir $IAEA_EXTENSION_DIR COPY setup/start_ckan_development.sh ${APP_DIR} COPY setup/start_ckan_worker.sh ${APP_DIR} +COPY ckan.ini ${APP_DIR} CMD ["/srv/app/start_ckan_development.sh"] +#CMD ["sleep", "10000"] diff --git a/docker/ckan/ckan.ini b/docker/ckan/ckan.ini new file mode 100644 index 0000000..546fe88 --- /dev/null +++ b/docker/ckan/ckan.ini @@ -0,0 +1,258 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 5000 + + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = LpgiJSdPY7uSMEiPHvX8jJzYA + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = f296bd5f-cff8-4180-b6a9-3f87bbfc4fbb + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +# who.timeout = 86400 + +## Database Settings +sqlalchemy.url = postgresql://ckan:ckan@iaea_db/ckan + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = http://ckan.iaea.local:5000 +#ckan.use_pylons_response_cleanup_middleware = true + +# Default timeout for Requests +#ckan.requests.timeout = 10 + + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = false +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = true +ckan.auth.public_activity_stream_detail = true +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:LpgiJSdPY7uSMEiPHvX8jJzYA +api_token.jwt.decode.secret = string:LpgiJSdPY7uSMEiPHvX8jJzYA +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default +#solr_url = http://127.0.0.1:8983/solr + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +#ckan.redis.url = redis://localhost:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +ckan.plugins = envvars stats text_view image_view recline_view + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +#ckan.storage_path = /var/lib/ckan +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 +# Default timeout for fetching proxied items +#ckan.resource_proxy.timeout = 10 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/docker/ckan/development.ini b/docker/ckan/development.ini deleted file mode 100644 index a97267d..0000000 --- a/docker/ckan/development.ini +++ /dev/null @@ -1,352 +0,0 @@ -# -# CKAN - Pylons configuration -# -# These are some of the configuration options available for your CKAN -# instance. Check the documentation in 'doc/configuration.rst' or at the -# following URL for a description of what they do and the full list of -# available options: -# -# http://docs.ckan.org/en/latest/maintaining/configuration.html -# -# The %(here)s variable will be replaced with the parent directory of this file -# - -[DEFAULT] - -# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* -debug = false - -[server:main] -use = egg:Paste#http -host = 0.0.0.0 -port = 5000 - -[app:main] -use = egg:ckan -full_stack = true -cache_dir = /tmp/%(ckan.site_id)s/ -beaker.session.key = ckan - -# Enable tracking -ckan.tracking_enabled = true - -# This is the secret token that the beaker library uses to hash the cookie sent -# to the client. `paster make-config` generates a unique value for this each -# time it generates a config file. -beaker.session.secret = p3yVBxMItfwQtsXOE++gZatJO - -# `paster make-config` generates a unique value for this each time it generates -# a config file. -app_instance_uuid = 08bfdc21-977a-4a83-ae15-4e8cfbc0120a - -# repoze.who config -who.config_file = /usr/lib/ckan/default/src/ckan/who.ini -who.log_level = warning -who.log_file = %(cache_dir)s/who_log.ini -# Session timeout (user logged out after period of inactivity, in seconds). -# Inactive by default, so the session doesn't expire. -# who.timeout = 86400 - -## Database Settings -sqlalchemy.url = postgresql://ckan:ckan@localhost:5432/ckan - -ckan.datastore.write_url = postgresql://ckan:ckan@localhost:5432/datastore -ckan.datastore.read_url = postgresql://datastore_ro:datastore@localhost:5432/datastore - -# PostgreSQL' full-text search parameters -ckan.datastore.default_fts_lang = english -ckan.datastore.default_fts_index_method = gist -ckan.datastore.sqlsearch.allowed_functions_file=/srv/app/allowed_functions.txt - - -## Site Settings - -ckan.site_url = http://ckan.iaea.local:5000 -#ckan.site_url = https://data-dev.iaea.org -#ckan.use_pylons_response_cleanup_middleware = true - -## Authorization Settings - -ckan.auth.anon_create_dataset = false -ckan.auth.create_unowned_dataset = true -ckan.auth.create_dataset_if_not_in_organization = false -ckan.auth.user_create_groups = false -ckan.auth.user_create_organizations = false -ckan.auth.user_delete_groups = true -ckan.auth.user_delete_organizations = true -ckan.auth.create_user_via_api = false -ckan.auth.create_user_via_web = true -ckan.auth.roles_that_cascade_to_sub_groups = admin - - -## Search Settings - -ckan.site_id = default -solr_url = http://127.0.0.1:8983/solr/ckan - - -## Redis Settings - -# URL to your Redis instance, including the database to be used. -ckan.redis.url = redis://localhost:6379/0 - - -## CORS Settings - -# If cors.origin_allow_all is true, all origins are allowed. -# If false, the cors.origin_whitelist is used. -# ckan.cors.origin_allow_all = true -# cors.origin_whitelist is a space separated list of allowed domains. -# ckan.cors.origin_whitelist = http://example1.com http://example2.com - - -## Plugins Settings - -# Note: Add ``datastore`` to enable the CKAN DataStore -# Add ``datapusher`` to enable DataPusher -# Add ``resource_proxy`` to enable resorce proxying and get around the -# same origin policy -#ckan.plugins = envvars image_view text_view recline_view webpage_view pdf_view pages datastore datapusher harvest ckan_harvester iaea iaea_db_harvester authz_service -#ckan.plugins = envvars image_view text_view recline_view -#ckan.plugins = envvars stats text_view recline_view image_view webpage_view resource_proxy datastore datapusher report archiver qa harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data iaea -#ckan.plugins = envvars stats text_view image_view webpage_view resource_proxy datastore xloader visualize iaea validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations saml2auth -ckan.plugins = envvars stats text_view image_view webpage_view resource_proxy datastore xloader validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations iaea saml2auth - -# Define which views should be created by default -# (plugins must be loaded in ckan.plugins) -#ckan.views.default_views = image_view text_view recline_view -ckan.views.default_views = image_view text_view recline_view pdf_view video_view geojson_view - -# Customize which text formats the text_view plugin will show -#ckan.preview.json_formats = json -#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss -#ckan.preview.text_formats = text plain text/plain - -# Customize which image formats the image_view plugin will show -#ckan.preview.image_formats = png jpeg jpg gif - -## Front-End Settings - -# Uncomment following configuration to enable using of Bootstrap 2 -#ckan.base_public_folder = public-bs2 -#ckan.base_templates_folder = templates-bs2 - -ckan.site_title = CKAN -ckan.site_logo = /base/images/ckan-logo.png -ckan.site_description = -ckan.favicon = /base/images/ckan.ico -ckan.gravatar_default = identicon -ckan.preview.direct = png jpg gif -ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json -ckan.display_timezone = server - -# package_hide_extras = for_search_index_only -#package_edit_return_url = http://another.frontend/dataset/ -#package_new_return_url = http://another.frontend/dataset/ -#ckan.recaptcha.publickey = -#ckan.recaptcha.privatekey = -#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json -# ckan.template_footer_end = - - -## Internationalisation Settings -ckan.locale_default = en -ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv -ckan.locales_offered = -ckan.locales_filtered_out = en_GB - -## Feeds Settings - -ckan.feeds.authority_name = -ckan.feeds.date = -ckan.feeds.author_name = -ckan.feeds.author_link = - -## Storage Settings - -ckan.storage_path = /usr/lib/ckan/iaea-storage -ckan.max_resource_size = 100 -#ckan.max_image_size = 2 - -## Datapusher settings - -# Make sure you have set up the DataStore - -#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -ckan.datapusher.url = http://127.0.0.1:8800/ -#ckan.datapusher.assume_task_stale_after = 3600 - -# Resource Proxy settings -# Preview size limit, default: 1MB -#ckan.resource_proxy.max_file_size = 1048576 -# Size of chunks to read/write. -#ckan.resource_proxy.chunk_size = 4096 - -## Activity Streams Settings - -#ckan.activity_streams_enabled = true -#ckan.activity_list_limit = 31 -#ckan.activity_streams_email_notifications = true -#ckan.email_notifications_since = 2 days -ckan.hide_activity_from_users = %(ckan.site_id)s - - -## Email settings - -#email_to = errors@example.com -#error_email_from = ckan-errors@example.com -#smtp.server = localhost -#smtp.starttls = False -#smtp.user = username@example.com -#smtp.password = your_password -#smtp.mail_from = - - -## Other extensions - -# Sentry -#sentry.dsn = https://xxxxxx:xxxxxx@sentry.domain.com/1 -#sentry.configure_logging=True -#sentry.log_level=WARN - -# Autz-Service -ckanext.authz_service.jwt_private_key_file=./test-keys/jwt-rs256.key -ckanext.authz_service.jwt_public_key_file=./test-keys/jwt-rs256.key.pub -ckanext.authz_service.jwt_algorithm=RS256 - - -# Googleanalytics -ckanext.iaea.googleanalytics.id=G-REPLACEME - -# Sets default ogranization for new datasets -ckanext.iaea.main_organization=iaea -ckanext.iaea.allow_dataset_create_from_organization=iaea,load_test - -## ckanext-saml2auth -# Specifies the metadata location type -# Options: local or remote -ckanext.saml2auth.idp_metadata.location = local - -# Path to a local file accessible on the server the service runs on -# Ignore this config if the idp metadata location is set to: remote -ckanext.saml2auth.idp_metadata.local_path = sso/sso_idp.xml - -# A remote URL serving aggregate metadata -# Ignore this config if the idp metadata location is set to: local -# ckanext.saml2auth.idp_metadata.remote_url = https://kalmar2.org/simplesaml/module.php/aggregator/?id=kalmarcentral2&set=saml2 - -# Path to a local file accessible on the server the service runs on -# Ignore this config if the idp metadata location is set to: local and metadata is public -# ckanext.saml2auth.idp_metadata.remote_cert = /opt/metadata/kalmar2.cert - -# Corresponding SAML user field for firstname -ckanext.saml2auth.user_firstname = FirstName - -# Corresponding SAML user field for lastname -ckanext.saml2auth.user_lastname = LastName - -# Corresponding SAML user field for fullname -# (Optional: Can be used as an alternative to firstname + lastname) -# ckanext.saml2auth.user_fullname = fullname - -# Corresponding SAML user field for email -ckanext.saml2auth.user_email = EmailAddress - -# URL route of the endpoint where the SAML assertion is sent, also known as Assertion Consumer Service (ACS). -# Default: /acs -ckanext.saml2auth.acs_endpoint = /sso/post - -# Configuration setting that enables CKAN's internal register/login functionality as well -# Default: False -ckanext.saml2auth.enable_ckan_internal_login = True - -ckanext.saml2auth.want_assertions_or_response_signed = False -ckanext.saml2auth.want_response_signed = False -ckanext.saml2auth.key_file_path=sso/iaea-dev.key -ckanext.saml2auth.cert_file_path=sso/iaea-dev.crt -ckanext.saml2auth.sp.name_id_format=urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified - -# Entity ID (also know as Issuer) -# Define the entity ID. Default is urn:mace:umu.se:saml:ckan:sp -ckanext.saml2auth.entity_id=https://data-dev.iaea.org - - -ckanext.saml2auth.user_default_org=orgone -ckanext.saml2auth.user_default_role=admin - -#ckanext.saml2auth.sysadmins_list= - -## CKAN Scheming -scheming.dataset_schemas = ckanext.iaea:iaea_dataset_schema.yaml -scheming.presets = ckanext.scheming:presets.json ckanext.iaea:presets.json ckanext.validation:presets.json - -## Xloader -ckanext.xloader.jobs_db.uri = sqlite:////tmp/xloader_jobs.db -ckanext.xloader.max_content_length = 5000000001 -ckanext.xloader.job_timeout=36000 -ckanext.xloader.max_excerpt_lines=0 -ckanext.xloader.ssl_verify=false -ckanext.xloader.formats=csv application/csv xls application/vnd.ms-excel xlsx application/vnd.oasis.opendocument.spreadsheet -ckanext.xloader.rewrite_site_url=http://ckan.iaea.local:5000 - -# Validation -ckanext.validation.formats=csv xlsx xls -ckanext.validation.run_on_create_async=True -ckanext.validation.run_on_update_async=True -ckanext.validation.run_on_create_sync=False -ckanext.validation.run_on_update_sync=False -ckanext.validation.show_badges_in_listings=True - -# Harvest -ckan.harvest.mq.type=redis -ckan.harvest.mq.hostname=localhost -ckan.harvest.mq.port=6379 -ckan.harvest.mq.db=0 - -# Pages -ckanext.pages.editor = ckeditor - -# Dcat -ckanext.dcat.enable_content_negotiation=True - - -## Logging configuration -[loggers] -keys = root, ckan, ckanext - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARNING -handlers = console - -[logger_ckan] -level = INFO -handlers = console -qualname = ckan -propagate = 0 - -[logger_ckanext] -level = DEBUG -handlers = console -qualname = ckanext -propagate = 0 - -[logger_saml2] -level = DEBUG -handlers = console -qualname = saml2 -propagate = 0 - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s \ No newline at end of file diff --git a/docker/ckan/setup/start_ckan_development.sh b/docker/ckan/setup/start_ckan_development.sh index 736ab09..0f31a9c 100755 --- a/docker/ckan/setup/start_ckan_development.sh +++ b/docker/ckan/setup/start_ckan_development.sh @@ -27,7 +27,7 @@ do if [ -f $i/setup.py ]; then cd $i - python $i/setup.py develop + python3 $i/setup.py develop echo "Found setup.py file in $i" cd $APP_DIR fi @@ -36,25 +36,25 @@ do if [ -f $i/test.ini ]; then echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" - paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" fi fi done # Set debug to true echo "Enabling debug mode" -paster --plugin=ckan config-tool $CKAN_INI -s DEFAULT "debug = true" +ckan config-tool $CKAN_INI -s DEFAULT "debug = true" # Update the plugins setting in the ini file with the values defined in the env var echo "Loading the following plugins: $CKAN__PLUGINS" -paster --plugin=ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" -paster --plugin=ckan config-tool $CKAN_INI "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" +ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" +ckan config-tool $CKAN_INI "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" -paster --plugin=ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" +ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" # Update test-core.ini DB, SOLR & Redis settings echo "Loading test settings into test-core.ini" -paster --plugin=ckan config-tool $SRC_DIR/ckan/test-core.ini \ +ckan config-tool $SRC_DIR/ckan/test-core.ini \ "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ @@ -62,13 +62,13 @@ paster --plugin=ckan config-tool $SRC_DIR/ckan/test-core.ini \ "ckan.redis.url = $TEST_CKAN_REDIS_URL" # Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python prerun.py +python3 prerun.py echo "Running DB init scripts" -paster --plugin=ckanext-archiver archiver init --config="$CKAN_INI" -paster --plugin=ckanext-report report initdb --config="$CKAN_INI" -paster --plugin=ckanext-qa qa init --config="$CKAN_INI" -paster --plugin=ckanext-validation validation init-db --config="$CKAN_INI" +# paster --plugin=ckanext-archiver archiver init --config="$CKAN_INI" +# paster --plugin=ckanext-report report initdb --config="$CKAN_INI" +# paster --plugin=ckanext-qa qa init --config="$CKAN_INI" +# paster --plugin=ckanext-validation validation init-db --config="$CKAN_INI" echo "Init DB scripts completed." # Run any startup scripts provided by images extending this one @@ -77,7 +77,7 @@ then for f in /docker-entrypoint.d/*; do case "$f" in *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; *) echo "$0: Ignoring $f (not an sh or py file)" ;; esac echo @@ -88,4 +88,4 @@ fi # supervisord --configuration /etc/supervisord.conf & # Start the development server with automatic reload -sudo -u ckan -EH paster serve --reload $CKAN_INI +ckan -c "${CKAN_INI}" run diff --git a/docker/ckan/setup/start_ckan_worker.sh b/docker/ckan/setup/start_ckan_worker.sh index 5819e41..53cb54e 100755 --- a/docker/ckan/setup/start_ckan_worker.sh +++ b/docker/ckan/setup/start_ckan_worker.sh @@ -27,7 +27,7 @@ do if [ -f $i/setup.py ]; then cd $i - python $i/setup.py develop + python3 $i/setup.py develop echo "Found setup.py file in $i" cd $APP_DIR fi @@ -36,7 +36,7 @@ do if [ -f $i/test.ini ]; then echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" - paster --plugin=ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" + ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" fi fi done @@ -44,13 +44,13 @@ done # Set debug to true echo "Enabling debug mode" -paster --plugin=ckan config-tool $CKAN_INI -s DEFAULT "debug = true" +ckan config-tool $CKAN_INI -s DEFAULT "debug = true" # Update the plugins setting in the ini file with the values defined in the env var echo "Loading the following plugins: $CKAN__PLUGINS" -paster --plugin=ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" +ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" -paster --plugin=ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" +ckan config-tool $CKAN_INI "ckan.site_url = $CKAN_SITE_URL" echo "Running worker: $1" -paster --plugin=ckan jobs worker "$1" -c "$CKAN_INI" \ No newline at end of file +ckan jobs worker "$1" -c "$CKAN_INI" \ No newline at end of file diff --git a/docker/postgresql/Dockerfile b/docker/postgresql/Dockerfile index f221611..ebb6341 100644 --- a/docker/postgresql/Dockerfile +++ b/docker/postgresql/Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:9.6-alpine +FROM postgres:14.11-alpine MAINTAINER Open Knowledge International # Allow connections; we don't map out any ports so only linked docker containers can connect diff --git a/docker/solr/Dockerfile b/docker/solr/Dockerfile deleted file mode 100644 index 2484739..0000000 --- a/docker/solr/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM solr:6.5.1-alpine -MAINTAINER Open Knowledge - -# Enviroment -ENV SOLR_CORE ckan -# Adjust to your needs -ENV CKAN_VERSION dev-v2.8 - -# User -USER root - -# Create Directories -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data - -# Adding Files -ADD solrconfig.xml \ -https://raw.githubusercontent.com/ckan/ckan/$CKAN_VERSION/ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/$SOLR_CORE/conf/ - -# Create Core.properties -RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties - -# Giving ownership to Solr -RUN chown -R solr:solr /opt/solr/server/solr/$SOLR_CORE - -USER solr diff --git a/docker/solr/solrconfig.xml b/docker/solr/solrconfig.xml deleted file mode 100644 index 9ac620c..0000000 --- a/docker/solr/solrconfig.xml +++ /dev/null @@ -1,343 +0,0 @@ - - - - - - 6.0.0 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - ${solr.lock.type:native} - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - ${solr.autoCommit.maxTime:15000} - false - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - 1024 - - - - - true - 20 - 200 - - - - - - - - - - false - 2 - - - - - - - - - - - - - - explicit - 10 - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - - - _text_ - - - - - - - - add-unknown-fields-to-the-schema - - - - - - - - true - ignored_ - _text_ - - - - - - - - - - - explicit - true - - - - - - - text_general - - - default - _text_ - solr.DirectSolrSpellChecker - internal - 0.5 - 2 - 1 - 5 - 4 - 0.01 - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - true - false - - - - terms - - - - - - - string - elevate.xml - - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - 70 - 0.5 - [-\w ,/\n\"']{20,200} - - - - - - ]]> - ]]> - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - - 10 - .,!? - - - - - - WORD - en - US - - - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - text/plain; charset=UTF-8 - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - 5 - - - From 144200ed041b7cc7846354c82aba595f7edea73b Mon Sep 17 00:00:00 2001 From: Pavle Jonoski Date: Sun, 10 Mar 2024 19:30:07 +0100 Subject: [PATCH 3/4] Sets up extensions in the docker file and docker compose. --- docker-compose.yml | 1 + docker/ckan/Dockerfile | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 35e767c..90a0123 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: "3" services: ckan: + profiles: ["full"] build: context: docker/ckan/ dockerfile: Dockerfile diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile index 3fa59d4..e1dc293 100644 --- a/docker/ckan/Dockerfile +++ b/docker/ckan/Dockerfile @@ -73,4 +73,3 @@ COPY ckan.ini ${APP_DIR} CMD ["/srv/app/start_ckan_development.sh"] -#CMD ["sleep", "10000"] From d73cd84f592379c7cb2960f9518f65e7b41690a0 Mon Sep 17 00:00:00 2001 From: Pavle Jonoski Date: Sun, 10 Mar 2024 19:32:33 +0100 Subject: [PATCH 4/4] Migrates dataset view templates and snippets. --- ckanext/iaea/middleware.py | 18 ++ ckanext/iaea/plugin.py | 127 ++++++++++- ckanext/iaea/templates/package/base.html | 28 +++ .../iaea/templates/package/features_view.html | 37 ++++ ckanext/iaea/templates/package/metadata.html | 6 + ckanext/iaea/templates/package/read.html | 11 + ckanext/iaea/templates/package/read_base.html | 156 ++++++++++++++ .../package/snippets/additional_info.html | 90 ++++++++ .../templates/package/snippets/view_form.html | 9 + .../snippets/view_form_field_filter.html | 22 ++ ckanext/iaea/view.py | 197 ++++++++++++++++++ 11 files changed, 693 insertions(+), 8 deletions(-) create mode 100644 ckanext/iaea/middleware.py create mode 100644 ckanext/iaea/templates/package/base.html create mode 100644 ckanext/iaea/templates/package/features_view.html create mode 100644 ckanext/iaea/templates/package/metadata.html create mode 100644 ckanext/iaea/templates/package/read.html create mode 100644 ckanext/iaea/templates/package/snippets/additional_info.html create mode 100644 ckanext/iaea/templates/package/snippets/view_form.html create mode 100644 ckanext/iaea/templates/package/snippets/view_form_field_filter.html create mode 100644 ckanext/iaea/view.py diff --git a/ckanext/iaea/middleware.py b/ckanext/iaea/middleware.py new file mode 100644 index 0000000..279b175 --- /dev/null +++ b/ckanext/iaea/middleware.py @@ -0,0 +1,18 @@ +class RestrictMiddleware(object): + + def __init__(self, app, app_config): + self.app=app + + def __call__(self, environ, start_response): + ui_path = environ.get('PATH_INFO') + + if ui_path == "/stats" and not 'repoze.who.identity' in environ: + + status = u'404 Not Found' + location = u'/user/login' + headers = [(u'Location',location), (u'Content-type', u'text/plain')] + body='Not authorized to see this page' + start_response (status, headers) + return[body] + else: + return self.app(environ, start_response) \ No newline at end of file diff --git a/ckanext/iaea/plugin.py b/ckanext/iaea/plugin.py index 85ad1c5..3d13be2 100644 --- a/ckanext/iaea/plugin.py +++ b/ckanext/iaea/plugin.py @@ -1,10 +1,29 @@ import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit +import ckan.logic as logic from ckan.lib.plugins import DefaultTranslation -from ckanext.iaea.helpers import get_helpers from ckanext.iaea.logic import action, validators -import ckan.logic as logic +import ckanext.iaea.logic.auth as ia +from flask import Blueprint +from ckanext.iaea import view import ckan.model as model +import ckanext.iaea.middleware as middleware +from ckan.model.meta import engine +#from threading import Thread, Event +#import signal +import sys +from logging import getLogger +import os + +#import ckanext.iaea.profiler as profiler + +from ckanext.iaea.helpers import get_helpers + + +def package_activity_html(id): + activity = logic.get_action( + 'package_activity_list_html')({}, {'id': id ,'limit': 8}) + return activity def featured_group(): @@ -20,11 +39,79 @@ def featured_group(): return {} -class IaeaPlugin(plugins.SingletonPlugin, DefaultTranslation): - plugins.implements(plugins.IConfigurer) +def suggested_filter_fields_serializer(datapackage, view_dict): + suggested_filter_fields = view_dict.get('suggested_filter_fields', False) + try: + fields = datapackage['resources'][0]['schema']['fields'] + except KeyError as e: + fields = [] + rules = [] + date = {} + if suggested_filter_fields: + suggested_fields_with_type = [field for field in fields if field['name'] in suggested_filter_fields] + for field in suggested_fields_with_type: + if field['type'] in ['datetime', 'date']: + date = { + 'startDate': None, + 'endDate': None, + 'fieldName': field['name'] + } + else: + rules.append({ + 'combinator': 'AND', + 'field': field['name'], + 'operator': '=', + 'value': '' + }) + if rules: + datapackage['resources'][0].update({'rules': rules}) + if date: + datapackage['resources'][0].update({'date': date}) + return datapackage + + +def featured_view_url(pkg): + featured_view = model.ResourceView.get(pkg['featured_view']) + return toolkit.h.url_for(qualified=True, controller='dataset_resource', + action='view', id=pkg['name'], + resource_id=featured_view.resource_id, + view_id=featured_view.id) + + +class IaeaPlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm, + DefaultTranslation): plugins.implements(plugins.ITranslation) - plugins.implements(plugins.ITemplateHelpers, inherit=True) + plugins.implements(plugins.IConfigurer) + plugins.implements(plugins.IDatasetForm) + plugins.implements(plugins.IActions) plugins.implements(plugins.IValidators) + plugins.implements(plugins.IBlueprint) + plugins.implements(plugins.ITemplateHelpers, inherit=True) + plugins.implements(plugins.IMiddleware, inherit=True) + plugins.implements(plugins.IAuthFunctions) + + # IDatasetForm + def update_package_schema(self): + schema = super(IaeaPlugin, self).update_package_schema() + schema.update({ + u'featured_view': [toolkit.get_validator(u'ignore_missing'), + toolkit.get_converter(u'convert_to_extras')] + }) + return schema + + def show_package_schema(self): + schema = super(IaeaPlugin, self).show_package_schema() + schema.update({ + u'featured_view': [toolkit.get_converter(u'convert_from_extras'), + toolkit.get_validator(u'ignore_missing')], + }) + return schema + + def is_fallback(self): + return True + + def package_types(self): + return [] # IConfigurer @@ -38,15 +125,39 @@ def update_config(self, config_): def get_helpers(self): iaea_helpers = { 'featured_group': featured_group, - # 'package_activity_html': package_activity_html, - # 'suggested_filter_fields_serializer': suggested_filter_fields_serializer, - # 'featured_view_url': featured_view_url, + 'package_activity_html': package_activity_html, + 'suggested_filter_fields_serializer': suggested_filter_fields_serializer, + 'featured_view_url': featured_view_url, } iaea_helpers.update(get_helpers()) return iaea_helpers + # IActions + def get_actions(self): + return { + 'resource_view_create': action.resource_view_create, + 'resource_view_update': action.resource_view_update, + } + # IValidators def get_validators(self): return { 'iaea_owner_org_validator': validators.package_organization_validator, } + + # IBlueprint + def get_blueprint(self): + blueprint = Blueprint(self.name, self.__module__) + blueprint.template_folder = u'templates' + # Add plugin url rules to Blueprint object + blueprint.add_url_rule(u'/dataset/metadata/', view_func=view.metadata) + blueprint.add_url_rule(u'/dataset//view', view_func=view.FeatureView.as_view(str(u'feature_view'))) + return blueprint + + # IAuthFunctions + def get_auth_functions(self): + return {'package_create': ia.package_create} + + def make_middleware(self, app, config): + + return middleware.RestrictMiddleware(app, config) diff --git a/ckanext/iaea/templates/package/base.html b/ckanext/iaea/templates/package/base.html new file mode 100644 index 0000000..2443fc0 --- /dev/null +++ b/ckanext/iaea/templates/package/base.html @@ -0,0 +1,28 @@ +{% extends "page.html" %} + +{% set pkg = c.pkg_dict or pkg_dict %} +{% set current_url = h.full_current_url() %} + +{% block breadcrumb_content_selected %} class="active"{% endblock %} + +{% block subtitle %}{{ _('Datasets') }}{% endblock %} + +{% block breadcrumb_content %} +{% if pkg %} + {% set dataset = h.dataset_display_name(pkg) %} + {% if pkg.organization %} + {% set organization = h.get_translated(pkg.organization, 'title') or pkg.organization.name %} + {% set group_type = pkg.organization.type %} +
  • {% link_for _('Organizations'), controller='organization', action='index', named_route=group_type + '_index' %}
    • +
  • {% link_for organization|truncate(30), controller='organization', action='read', id=pkg.organization.name, + named_route=group_type + '_read' %}
    • + {% else %} +
  • {% link_for _('Datasets'), controller='dataset', action='search' %}
    • +{% endif %} + {% link_for dataset|truncate(30), controller='dataset', action='read', + id=pkg.name %} +{% else %} +
  • {% link_for _('Datasets'), controller='dataset', action='search' %}
    • +
  • {{ _('Create Dataset') }}
    • +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/features_view.html b/ckanext/iaea/templates/package/features_view.html new file mode 100644 index 0000000..17f79b1 --- /dev/null +++ b/ckanext/iaea/templates/package/features_view.html @@ -0,0 +1,37 @@ +{% extends "package/read_base.html" %} + +{% block primary_content_inner %} +{% if package_views %} + +
    +
    +

    Select a view that should be featured on the dataset page.

    + {% for resource in package_views %} +

    {{ resource.resource_name }}

    + {% for view in resource.views %} +
    + {% set view_url = h.url_for(qualified=True, controller='dataset_resource', + action='view', id=pkg['name'], + resource_id=view.resource_id, + view_id=view.id) %} + {% if pkg['featured_view'] == view.id %} + + {{view.title}} + {% else %} + + {{view.title}} + {% endif %} +
    + {% endfor %} + {% endfor %} +
    + +
    + + +
    +
    +{% else %} +

    {{ _("There are no views created for this dataset yet.") }}

    +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/metadata.html b/ckanext/iaea/templates/package/metadata.html new file mode 100644 index 0000000..290289c --- /dev/null +++ b/ckanext/iaea/templates/package/metadata.html @@ -0,0 +1,6 @@ +{% extends "package/read_base.html" %} +{# Reuse of activity template for additonal info#} + +{% block primary_content_inner %} +{% snippet "package/snippets/additional_info.html", pkg_dict=pkg %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/read.html b/ckanext/iaea/templates/package/read.html new file mode 100644 index 0000000..c349485 --- /dev/null +++ b/ckanext/iaea/templates/package/read.html @@ -0,0 +1,11 @@ +{% ckan_extends %} + +{% block package_description %} +{% endblock %} + +{% block package_resources %} + {% snippet "package/snippets/resources_list.html", pkg=pkg, resources=pkg.resources %} +{% endblock %} + +{% block package_additional_info %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/read_base.html b/ckanext/iaea/templates/package/read_base.html index d224d03..aee1a48 100644 --- a/ckanext/iaea/templates/package/read_base.html +++ b/ckanext/iaea/templates/package/read_base.html @@ -1,4 +1,160 @@ {% ckan_extends %} + +{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ super() }}{% endblock %} + +{% block head_extras -%} + {{ super() }} + {% set description = h.markdown_extract(pkg.notes, extract_length=200)|forceescape %} + + +{% endblock -%} + + + +{% block content_action %} + {% if h.check_access('package_update', {'id':pkg.id }) %} + {% link_for _('Manage'), controller='dataset', action='edit', id=pkg.name, class_='btn btn-default', icon='wrench' %} + {% endif %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name) }} + {{ h.build_nav_icon('iaea.metadata', _('Metadata'), id=pkg.name, icon='fa fa-tag') }} + {% if h.check_access('package_update', {'id':pkg.id }) %} + {{ h.build_nav_icon('iaea.feature_view', _('View'), id=pkg.name, icon='check-square-o') }} + {% endif %} +{% endblock %} + +{% block primary_content_inner %} + {% block package_revision_info %} + {% if c.revision_date %} +
    +

    + {% set timestamp = h.render_datetime(c.revision_date, with_hours=True) %} + {% set url = h.url_for(controller='dataset', action='read', id=pkg.name) %} + + {% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the current revision.{% endtrans %} +

    +
    + {% endif %} + {% endblock %} +{% endblock %} +{% block pre_primary %} +
    +
    +
    + {% if pkg.private %} + + + {{ _('Private') }} + + {% endif %} +

    + {% block page_heading %} + {{ h.dataset_display_name(pkg) }} + {% if pkg.state.startswith('draft') %} + [{{ _('Draft') }}] + {% endif %} + {% if pkg.state == 'deleted' %} + [{{ _('Deleted') }}] + {% endif %} + {% endblock %} +

    + {% block package_notes %} + {% if pkg.notes %} +
    + {{ h.render_markdown(h.get_translated(pkg, 'notes')) }} +
    + {% endif %} + {% endblock %} +
    +
    +
    + {% block title_and_actions_right_side %} + {% if pkg.organization %} + {% set org = h.get_organization(pkg.organization.name) %} + {% set url_org = h.url_for(pkg.organization.type + '_read', id=pkg.organization.name, ) %} +
    +
    + + {{ pkg.organization.name }} + +
    +
    + {% endif %} + {% endblock %} +
    +
    +
    +
    +
    + Last Update: + {{pkg.metadata_modified}} + +
    +
    + {% if pkg.maintainer_email %} +
    + + + Contact dataset maintainer + +
    + {% endif %} + +
    +
    +
    + {% if pkg.featured_view %} +
    +

    Featured Visualization

    +
    + + + Fullscreen + +
    +
    + + {% endif %} +{% endblock %} + + +{% block secondary_content %} +
    + {% block secondary_help_content %} +
    +
    +

    {{ _('Activity') }}

    +
    +
    +
    + {#{ h.package_activity_html(pkg.id) | safe }#} + {% endblock %} + + {% block package_organization %} + {% if pkg.organization %} + {% set org = h.get_organization(pkg.organization.id) %} + {% snippet "snippets/organization.html", organization=org, has_context_title=true %} + {% endif %} + {% endblock %} + + + {% block package_info %} + + {% endblock %} + + {% block package_license %} + {% endblock %} +
    +{% endblock %} + {% block main_content %} {{ super() }}
    diff --git a/ckanext/iaea/templates/package/snippets/additional_info.html b/ckanext/iaea/templates/package/snippets/additional_info.html new file mode 100644 index 0000000..cf33483 --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/additional_info.html @@ -0,0 +1,90 @@ +
    + + + + + + + + + {% block package_additional_info %} + {% if pkg_dict.url %} + + + {% if h.is_url(pkg_dict.url) %} + + {% else %} + + {% endif %} + + {% endif %} + + {% if pkg_dict.author_email %} + + + + + {% elif pkg_dict.author %} + + + + + {% endif %} + + {% if pkg_dict.maintainer_email %} + + + + + {% elif pkg_dict.maintainer %} + + + + + {% endif %} + + {% if pkg_dict.version %} + + + + + {% endif %} + + {% if h.check_access('package_update',{'id':pkg_dict.id}) %} + + + + + {% endif %} + {% if pkg_dict.metadata_modified %} + + + + + {% endif %} + {% if pkg_dict.metadata_created %} + + + + + + {% endif %} + + {% block extras scoped %} + {% for extra in h.sorted_extras(pkg_dict.extras) %} + {% set key, value = extra %} + + + + + {% endfor %} + {% endblock %} + + {% endblock %} + +
    {{ _('Field') }}{{ _('Value') }}
    {{ _('Source') }}{{ h.link_to(pkg_dict.url, pkg_dict.url, rel='foaf:homepage', target='_blank') }}{{ pkg_dict.url }}
    {{ _("Author") }}{{ h.mail_to(email_address=pkg_dict.author_email, name=pkg_dict.author) }}
    {{ _("Author") }}{{ pkg_dict.author }}
    {{ _('Maintainer') }}{{ h.mail_to(email_address=pkg_dict.maintainer_email, name=pkg_dict.maintainer) }}
    {{ _('Maintainer') }}{{ pkg_dict.maintainer }}
    {{ _("Version") }}{{ pkg_dict.version }}
    {{ _("State") }}{{ _(pkg_dict.state) }}
    {{ _("Last Updated") }} + {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_modified %} +
    {{ _("Created") }} + {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_created %} +
    {{ _(key) }}{{ value }}
    +
    \ No newline at end of file diff --git a/ckanext/iaea/templates/package/snippets/view_form.html b/ckanext/iaea/templates/package/snippets/view_form.html new file mode 100644 index 0000000..6f6af3d --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/view_form.html @@ -0,0 +1,9 @@ +{% ckan_extends %} + +{% block view_form_filters %} + {% if h.resource_view_is_filterable(resource_view) %} + {% snippet 'package/snippets/view_form_filters.html', resource=resource, resource_view=resource_view %} + {% endif %} + {% snippet 'package/snippets/view_form_field_filter.html', resource=resource, resource_view=resource_view, data=data, errors=errors %} + {# resource 'iaea/view_js' #} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/snippets/view_form_field_filter.html b/ckanext/iaea/templates/package/snippets/view_form_field_filter.html new file mode 100644 index 0000000..4e815aa --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/view_form_field_filter.html @@ -0,0 +1,22 @@ +{% import 'macros/form.html' as form %} +{% macro suggestedFilter(name, fields, suggested_filter_fields, label = '', error = '', options = '', classes = [], is_required = false) %} + {% set classes = (classes | list) %} + {%- set extra_html = caller() if caller -%} + {% call form.input_block(id or name, label or name, error, classes, extra_html = extra_html, is_required = is_required) %} +
    + +
    + {% endcall %} +{% endmacro %} + +{% set fields = h.resource_view_get_fields(resource) %} +
    + {{ + suggestedFilter('Suggested filter fields', label=_('Suggested filter fields'), fields=fields, suggested_filter_fields=data.suggested_filter_fields, error=errors, classes=[""]) + }} +
    \ No newline at end of file diff --git a/ckanext/iaea/view.py b/ckanext/iaea/view.py new file mode 100644 index 0000000..cea03ca --- /dev/null +++ b/ckanext/iaea/view.py @@ -0,0 +1,197 @@ +import logging +import ckan.logic as logic +import ckan.lib.base as base +import ckan.model as model +from flask.views import MethodView +import ckan.lib.dictization.model_dictize as model_dictize +from itertools import groupby +import ckan.lib.helpers as h + +from ckan.common import _, c, request + +log = logging.getLogger(__name__) + +render = base.render +abort = base.abort + +NotFound = logic.NotFound +NotAuthorized = logic.NotAuthorized +ValidationError = logic.ValidationError +check_access = logic.check_access +get_action = logic.get_action + + +def metadata(id): # noqa + """Render this package's public activity stream page. + """ + context = { + u'model': model, + u'session': model.Session, + u'user': c.user, + u'for_view': True, + u'auth_user_obj': c.userobj + } + + data_dict = {u'id': id} + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + # TODO: remove + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/metadata.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + } + ) + + +class FeatureView(MethodView): + """ feature selected view on dataset page + """ + + def _prepare(self, id): + context = { + u'model': model, + u'session': model.Session, + u'user': c.user, + u'auth_user_obj': c.userobj, + u'save': u'save' in request.form + } + try: + check_access(u'package_update', context, {u'id': id}) + + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort( + 403, + _(u'Unauthorized to edit %s') % u'' + ) + + return context + + def post(self, id): + context = self._prepare(id) + + featured_view = request.form.get('featured_view', False) + data_dict = {u'id': id} + if request.form['submit'] == 'submit': + if not featured_view: + h.flash_error('Please select view from the list.') + data_dict['featured_view'] = '' + else: + data_dict['featured_view'] = featured_view + else: + data_dict['featured_view'] = '' + + # update package with selected featured view + try: + pkg_dict = get_action(u'package_patch')(context, data_dict) + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + package_views = model.Session.query( + model.ResourceView + ).join(model.Resource).filter(model.Resource.package_id == pkg_dict['id']).all() + + package_views_list = model_dictize.resource_view_list_dictize( + package_views, context) + + package_views_dict = [] + package_views_list = sorted(package_views_list, key=lambda k: k['resource_id']) + for k, v in groupby(package_views_list, key=lambda x: x['resource_id']): + [resource_name, state ] = model.Session.query(model.Resource.name.label('name'), model.Resource.state.label('state')).filter( + model.Resource.id == k).first() + + if state == 'deleted': + continue + else: + view_dict = { + 'resource_name': resource_name, + 'resource_id': k, + 'views': list(v) + } + + package_views_dict.append(view_dict) + + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/features_view.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + u'package_views': package_views_dict + }) + + def get(self, id): + context = self._prepare(id) + data_dict = {u'id': id} + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + package_views = model.Session.query( + model.ResourceView + ).join(model.Resource).filter(model.Resource.package_id == pkg_dict['id']).all() + + package_views_list = model_dictize.resource_view_list_dictize( + package_views, context) + + package_views_dict = [] + package_views_list = sorted(package_views_list, key=lambda k: k['resource_id']) + for k, v in groupby(package_views_list, key=lambda x: x['resource_id']): + [resource_name, state ] = model.Session.query(model.Resource.name.label('name'), model.Resource.state.label('state')).filter( + model.Resource.id == k).first() + + if state == 'deleted': + continue + else: + view_dict = { + 'resource_name': resource_name, + 'resource_id': k, + 'views': list(v) + } + + package_views_dict.append(view_dict) + + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/features_view.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + u'package_views': package_views_dict + }) \ No newline at end of file