Merge pull request #35 from keitaroinc/migrate-docker-and-scripts

Migrate docker, scripts, package view and actions

.env

@@ -0,0 +1,56 @@
+# DB image settings
+POSTGRES_PASSWORD=ckan
+DATASTORE_READONLY_PASSWORD=datastore
+
+# Basic
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://ckan.iaea.local:5000
+CKAN_PORT=5000
+CKAN_SYSADMIN_NAME=ckan_admin
+CKAN_SYSADMIN_PASSWORD=test1234
+CKAN_SYSADMIN_EMAIL=your_email@example.com
+TZ=UTC
+
+# Database connections (TODO: avoid duplication)
+CKAN_SQLALCHEMY_URL=postgresql://ckan:ckan@iaea_db/ckan
+CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@iaea_db/datastore
+CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@iaea_db/datastore
+
+# Test database connections
+TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@iaea_db/ckan_test
+TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@iaea_db/datastore_test
+TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@iaea_db/datastore_test
+
+# Other services connections
+CKAN_SOLR_URL=http://iaea_solr:8983/solr/ckan
+CKAN_REDIS_URL=redis://iaea_redis:6379/1
+#CKAN_DATAPUSHER_URL=http://datapusher:8800
+#CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
+
+TEST_CKAN_SOLR_URL=http://iaea_solr:8983/solr/ckan
+TEST_CKAN_REDIS_URL=redis://iaea_redis:6379/1
+
+# Core settings
+CKAN__STORAGE_PATH=/var/lib/ckan
+
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+## Extensions
+#CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore datapusher iaea validation qa report archiver harvest ckan_harvester authz_service sentry linechart barchart piechart basicgrid visualize pdf_view geo_view geojson_view wmts_view shp_view pages dataexplorer_view dataexplorer_table_view dataexplorer_chart_view dataexplorer_map_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data scheming_datasets scheming_groups scheming_organizations
+CKAN__PLUGINS=envvars stats text_view image_view webpage_view resource_proxy datastore
+#CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view geojson_view
+CKAN__VIEWS__DEFAULT_VIEWS = image_view text_view recline_view
+CKAN__HARVEST__MQ__TYPE=redis
+CKAN__HARVEST__MQ__HOSTNAME=iaea_redis
+CKAN__HARVEST__MQ__PORT=6379
+CKAN__HARVEST__MQ__REDIS_DB=1
+
+# Sentry
+# CKAN___SENTRY__DSN=https://xxxxxx:xxxxxx@sentry.domain.com/1
+# CKAN___SENTRY__CONFIGURE_LOGGING=True
+# CKAN___SENTRY__LOG_LEVEL=WARN
+

ckanext/iaea/middleware.py

@@ -0,0 +1,18 @@
+class RestrictMiddleware(object):
+
+    def __init__(self, app, app_config):
+        self.app=app
+
+    def __call__(self, environ, start_response):
+        ui_path = environ.get('PATH_INFO')
+
+        if ui_path == "/stats" and not 'repoze.who.identity' in environ:
+
+            status = u'404 Not Found'
+            location = u'/user/login'
+            headers = [(u'Location',location), (u'Content-type', u'text/plain')]
+            body='Not authorized to see this page'
+            start_response (status, headers)  
+            return[body]
+        else:
+            return self.app(environ, start_response)

ckanext/iaea/plugin.py

@@ -1,10 +1,29 @@
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as toolkit
+import ckan.logic as logic
 from ckan.lib.plugins import DefaultTranslation
-from ckanext.iaea.helpers import get_helpers
 from ckanext.iaea.logic import action, validators
-import ckan.logic as logic
+import ckanext.iaea.logic.auth as ia
+from flask import Blueprint
+from ckanext.iaea import view
 import ckan.model as model
+import ckanext.iaea.middleware as middleware
+from ckan.model.meta import engine
+#from threading import Thread, Event
+#import signal
+import sys
+from logging import getLogger
+import os
+
+#import ckanext.iaea.profiler as profiler
+
+from ckanext.iaea.helpers import get_helpers
+
+
+def package_activity_html(id):
+    activity =  logic.get_action(
+            'package_activity_list_html')({}, {'id': id ,'limit': 8})
+    return activity
 
 
 def featured_group():
@@ -20,11 +39,79 @@ def featured_group():
         return {}
 
 
-class IaeaPlugin(plugins.SingletonPlugin, DefaultTranslation):
-    plugins.implements(plugins.IConfigurer)
+def suggested_filter_fields_serializer(datapackage, view_dict):
+    suggested_filter_fields = view_dict.get('suggested_filter_fields', False)
+    try:
+        fields = datapackage['resources'][0]['schema']['fields']
+    except KeyError as e:
+        fields = []
+    rules = []
+    date  = {}
+    if suggested_filter_fields:
+        suggested_fields_with_type = [field for field in fields if field['name'] in suggested_filter_fields]
+        for field in suggested_fields_with_type:
+            if field['type'] in ['datetime', 'date']:
+                date = {
+                    'startDate': None,
+                    'endDate': None,
+                    'fieldName': field['name']
+                }
+            else:
+                rules.append({
+                    'combinator': 'AND',
+                    'field': field['name'],
+                    'operator': '=',
+                    'value': ''
+                })
+    if rules:
+        datapackage['resources'][0].update({'rules': rules})
+    if date:
+        datapackage['resources'][0].update({'date': date})
+    return datapackage
+
+
+def featured_view_url(pkg):
+    featured_view = model.ResourceView.get(pkg['featured_view'])
+    return toolkit.h.url_for(qualified=True, controller='dataset_resource',
+                               action='view', id=pkg['name'],
+                               resource_id=featured_view.resource_id,
+                               view_id=featured_view.id)
+
+
+class IaeaPlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm,
+                  DefaultTranslation):
     plugins.implements(plugins.ITranslation)
-    plugins.implements(plugins.ITemplateHelpers, inherit=True)
+    plugins.implements(plugins.IConfigurer)
+    plugins.implements(plugins.IDatasetForm)
+    plugins.implements(plugins.IActions)
     plugins.implements(plugins.IValidators)
+    plugins.implements(plugins.IBlueprint)
+    plugins.implements(plugins.ITemplateHelpers, inherit=True)
+    plugins.implements(plugins.IMiddleware, inherit=True)
+    plugins.implements(plugins.IAuthFunctions)
+
+    # IDatasetForm
+    def update_package_schema(self):
+        schema = super(IaeaPlugin, self).update_package_schema()
+        schema.update({
+            u'featured_view': [toolkit.get_validator(u'ignore_missing'),
+                             toolkit.get_converter(u'convert_to_extras')]
+        })
+        return schema
+
+    def show_package_schema(self):
+        schema = super(IaeaPlugin, self).show_package_schema()
+        schema.update({
+            u'featured_view': [toolkit.get_converter(u'convert_from_extras'),
+                             toolkit.get_validator(u'ignore_missing')],
+        })
+        return schema
+
+    def is_fallback(self):
+        return True
+
+    def package_types(self):
+        return []
 
     # IConfigurer
 
@@ -38,15 +125,39 @@ def update_config(self, config_):
     def get_helpers(self):
         iaea_helpers = {
             'featured_group': featured_group,
-            # 'package_activity_html': package_activity_html,
-            # 'suggested_filter_fields_serializer': suggested_filter_fields_serializer,
-            # 'featured_view_url': featured_view_url,
+            'package_activity_html': package_activity_html,
+            'suggested_filter_fields_serializer': suggested_filter_fields_serializer,
+            'featured_view_url': featured_view_url,
         }
         iaea_helpers.update(get_helpers())
         return iaea_helpers
 
+    # IActions
+    def get_actions(self):
+        return {
+            'resource_view_create': action.resource_view_create,
+            'resource_view_update': action.resource_view_update,
+        }
+
     # IValidators
     def get_validators(self):
         return {
             'iaea_owner_org_validator': validators.package_organization_validator,
         }
+
+    # IBlueprint
+    def get_blueprint(self):
+        blueprint = Blueprint(self.name, self.__module__)
+        blueprint.template_folder = u'templates'
+        # Add plugin url rules to Blueprint object
+        blueprint.add_url_rule(u'/dataset/metadata/<id>', view_func=view.metadata)
+        blueprint.add_url_rule(u'/dataset/<id>/view', view_func=view.FeatureView.as_view(str(u'feature_view')))
+        return blueprint
+
+    # IAuthFunctions
+    def get_auth_functions(self):
+        return {'package_create': ia.package_create}
+
+    def make_middleware(self, app, config):
+
+        return middleware.RestrictMiddleware(app, config)

ckanext/iaea/templates/package/base.html

@@ -0,0 +1,28 @@
+{% extends "page.html" %}
+
+{% set pkg = c.pkg_dict or pkg_dict %}
+{% set current_url = h.full_current_url() %}
+
+{% block breadcrumb_content_selected %} class="active"{% endblock %}
+
+{% block subtitle %}{{ _('Datasets') }}{% endblock %}
+
+{% block breadcrumb_content %}
+{% if pkg %}
+    {% set dataset = h.dataset_display_name(pkg) %}
+    {% if pkg.organization %}
+        {% set organization = h.get_translated(pkg.organization, 'title') or pkg.organization.name %}
+        {% set group_type = pkg.organization.type %}
+        <li>{% link_for _('Organizations'), controller='organization', action='index', named_route=group_type + '_index' %}</li>
+        <li>{% link_for organization|truncate(30), controller='organization', action='read', id=pkg.organization.name,
+            named_route=group_type + '_read' %}</li>
+    {% else %}
+    <li>{% link_for _('Datasets'), controller='dataset', action='search' %}</li>
+{% endif %}
+    <li{{ self.breadcrumb_content_selected() }}>{% link_for dataset|truncate(30), controller='dataset', action='read',
+        id=pkg.name %}</li>
+{% else %}
+  <li>{% link_for _('Datasets'), controller='dataset', action='search' %}</li>
+  <li class="active"><a href="">{{ _('Create Dataset') }}</a></li>
+{% endif %}
+{% endblock %}

ckanext/iaea/templates/package/features_view.html

@@ -0,0 +1,37 @@
+{% extends "package/read_base.html" %}
+
+{% block primary_content_inner %}
+{% if package_views %}
+
+<form action="#" method="post">
+    <fieldset class="fields-feature-view form-group">
+        <p>Select a view that should be featured on the dataset page.</p>
+        {% for resource in package_views %}
+        <h3> {{ resource.resource_name }}</h3>
+        {% for view in resource.views %}
+        <div class="radio-groups">
+            {% set view_url = h.url_for(qualified=True, controller='dataset_resource',
+            action='view', id=pkg['name'],
+            resource_id=view.resource_id,
+            view_id=view.id) %}
+            {% if pkg['featured_view'] == view.id %}
+            <input id="featured_view" name="featured_view" type="radio" class="radio" value="{{view.id}}" checked />
+            <span><a href="{{view_url}}" target="_blank">{{view.title}}</a></span>
+            {% else %}
+            <input id="featured_view" name="featured_view" type="radio" class="radio" value="{{view.id}}" /> 
+            <span><a href="{{view_url}}" target="_blank">{{view.title}}</a></span>
+            {% endif %}
+        </div>
+        {% endfor %}
+        {% endfor %}
+    </fieldset>
+
+    <div class="form-actions">
+        <input type="submit" class="btn btn-primary" name="submit" value="submit" />
+        <input type="submit" class="btn btn-danger" name="submit" value="clear" />
+    </div>
+</form>
+{% else %}
+<p>{{ _("There are no views created for this dataset yet.") }}</p>
+{% endif %}
+{% endblock %}

ckanext/iaea/templates/package/metadata.html

@@ -0,0 +1,6 @@
+{% extends "package/read_base.html" %}
+{# Reuse of activity template for additonal info#}
+
+{% block primary_content_inner %}
+{% snippet "package/snippets/additional_info.html", pkg_dict=pkg %}
+{% endblock %}

ckanext/iaea/templates/package/read.html

@@ -0,0 +1,11 @@
+{% ckan_extends %}
+
+{% block package_description %}
+{% endblock %}
+
+{% block package_resources %}
+  {% snippet "package/snippets/resources_list.html", pkg=pkg, resources=pkg.resources %}
+{% endblock %}
+
+{% block package_additional_info %}
+{% endblock %}