diff --git a/ckanext/iaea/middleware.py b/ckanext/iaea/middleware.py new file mode 100644 index 0000000..279b175 --- /dev/null +++ b/ckanext/iaea/middleware.py @@ -0,0 +1,18 @@ +class RestrictMiddleware(object): + + def __init__(self, app, app_config): + self.app=app + + def __call__(self, environ, start_response): + ui_path = environ.get('PATH_INFO') + + if ui_path == "/stats" and not 'repoze.who.identity' in environ: + + status = u'404 Not Found' + location = u'/user/login' + headers = [(u'Location',location), (u'Content-type', u'text/plain')] + body='Not authorized to see this page' + start_response (status, headers) + return[body] + else: + return self.app(environ, start_response) \ No newline at end of file diff --git a/ckanext/iaea/plugin.py b/ckanext/iaea/plugin.py index 85ad1c5..3d13be2 100644 --- a/ckanext/iaea/plugin.py +++ b/ckanext/iaea/plugin.py @@ -1,10 +1,29 @@ import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit +import ckan.logic as logic from ckan.lib.plugins import DefaultTranslation -from ckanext.iaea.helpers import get_helpers from ckanext.iaea.logic import action, validators -import ckan.logic as logic +import ckanext.iaea.logic.auth as ia +from flask import Blueprint +from ckanext.iaea import view import ckan.model as model +import ckanext.iaea.middleware as middleware +from ckan.model.meta import engine +#from threading import Thread, Event +#import signal +import sys +from logging import getLogger +import os + +#import ckanext.iaea.profiler as profiler + +from ckanext.iaea.helpers import get_helpers + + +def package_activity_html(id): + activity = logic.get_action( + 'package_activity_list_html')({}, {'id': id ,'limit': 8}) + return activity def featured_group(): @@ -20,11 +39,79 @@ def featured_group(): return {} -class IaeaPlugin(plugins.SingletonPlugin, DefaultTranslation): - plugins.implements(plugins.IConfigurer) +def suggested_filter_fields_serializer(datapackage, view_dict): + suggested_filter_fields = view_dict.get('suggested_filter_fields', False) + try: + fields = datapackage['resources'][0]['schema']['fields'] + except KeyError as e: + fields = [] + rules = [] + date = {} + if suggested_filter_fields: + suggested_fields_with_type = [field for field in fields if field['name'] in suggested_filter_fields] + for field in suggested_fields_with_type: + if field['type'] in ['datetime', 'date']: + date = { + 'startDate': None, + 'endDate': None, + 'fieldName': field['name'] + } + else: + rules.append({ + 'combinator': 'AND', + 'field': field['name'], + 'operator': '=', + 'value': '' + }) + if rules: + datapackage['resources'][0].update({'rules': rules}) + if date: + datapackage['resources'][0].update({'date': date}) + return datapackage + + +def featured_view_url(pkg): + featured_view = model.ResourceView.get(pkg['featured_view']) + return toolkit.h.url_for(qualified=True, controller='dataset_resource', + action='view', id=pkg['name'], + resource_id=featured_view.resource_id, + view_id=featured_view.id) + + +class IaeaPlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm, + DefaultTranslation): plugins.implements(plugins.ITranslation) - plugins.implements(plugins.ITemplateHelpers, inherit=True) + plugins.implements(plugins.IConfigurer) + plugins.implements(plugins.IDatasetForm) + plugins.implements(plugins.IActions) plugins.implements(plugins.IValidators) + plugins.implements(plugins.IBlueprint) + plugins.implements(plugins.ITemplateHelpers, inherit=True) + plugins.implements(plugins.IMiddleware, inherit=True) + plugins.implements(plugins.IAuthFunctions) + + # IDatasetForm + def update_package_schema(self): + schema = super(IaeaPlugin, self).update_package_schema() + schema.update({ + u'featured_view': [toolkit.get_validator(u'ignore_missing'), + toolkit.get_converter(u'convert_to_extras')] + }) + return schema + + def show_package_schema(self): + schema = super(IaeaPlugin, self).show_package_schema() + schema.update({ + u'featured_view': [toolkit.get_converter(u'convert_from_extras'), + toolkit.get_validator(u'ignore_missing')], + }) + return schema + + def is_fallback(self): + return True + + def package_types(self): + return [] # IConfigurer @@ -38,15 +125,39 @@ def update_config(self, config_): def get_helpers(self): iaea_helpers = { 'featured_group': featured_group, - # 'package_activity_html': package_activity_html, - # 'suggested_filter_fields_serializer': suggested_filter_fields_serializer, - # 'featured_view_url': featured_view_url, + 'package_activity_html': package_activity_html, + 'suggested_filter_fields_serializer': suggested_filter_fields_serializer, + 'featured_view_url': featured_view_url, } iaea_helpers.update(get_helpers()) return iaea_helpers + # IActions + def get_actions(self): + return { + 'resource_view_create': action.resource_view_create, + 'resource_view_update': action.resource_view_update, + } + # IValidators def get_validators(self): return { 'iaea_owner_org_validator': validators.package_organization_validator, } + + # IBlueprint + def get_blueprint(self): + blueprint = Blueprint(self.name, self.__module__) + blueprint.template_folder = u'templates' + # Add plugin url rules to Blueprint object + blueprint.add_url_rule(u'/dataset/metadata/', view_func=view.metadata) + blueprint.add_url_rule(u'/dataset//view', view_func=view.FeatureView.as_view(str(u'feature_view'))) + return blueprint + + # IAuthFunctions + def get_auth_functions(self): + return {'package_create': ia.package_create} + + def make_middleware(self, app, config): + + return middleware.RestrictMiddleware(app, config) diff --git a/ckanext/iaea/templates/package/base.html b/ckanext/iaea/templates/package/base.html new file mode 100644 index 0000000..2443fc0 --- /dev/null +++ b/ckanext/iaea/templates/package/base.html @@ -0,0 +1,28 @@ +{% extends "page.html" %} + +{% set pkg = c.pkg_dict or pkg_dict %} +{% set current_url = h.full_current_url() %} + +{% block breadcrumb_content_selected %} class="active"{% endblock %} + +{% block subtitle %}{{ _('Datasets') }}{% endblock %} + +{% block breadcrumb_content %} +{% if pkg %} + {% set dataset = h.dataset_display_name(pkg) %} + {% if pkg.organization %} + {% set organization = h.get_translated(pkg.organization, 'title') or pkg.organization.name %} + {% set group_type = pkg.organization.type %} +
  • {% link_for _('Organizations'), controller='organization', action='index', named_route=group_type + '_index' %}
    • +
  • {% link_for organization|truncate(30), controller='organization', action='read', id=pkg.organization.name, + named_route=group_type + '_read' %}
    • + {% else %} +
  • {% link_for _('Datasets'), controller='dataset', action='search' %}
    • +{% endif %} + {% link_for dataset|truncate(30), controller='dataset', action='read', + id=pkg.name %} +{% else %} +
  • {% link_for _('Datasets'), controller='dataset', action='search' %}
    • +
  • {{ _('Create Dataset') }}
    • +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/features_view.html b/ckanext/iaea/templates/package/features_view.html new file mode 100644 index 0000000..17f79b1 --- /dev/null +++ b/ckanext/iaea/templates/package/features_view.html @@ -0,0 +1,37 @@ +{% extends "package/read_base.html" %} + +{% block primary_content_inner %} +{% if package_views %} + +
    +
    +

    Select a view that should be featured on the dataset page.

    + {% for resource in package_views %} +

    {{ resource.resource_name }}

    + {% for view in resource.views %} +
    + {% set view_url = h.url_for(qualified=True, controller='dataset_resource', + action='view', id=pkg['name'], + resource_id=view.resource_id, + view_id=view.id) %} + {% if pkg['featured_view'] == view.id %} + + {{view.title}} + {% else %} + + {{view.title}} + {% endif %} +
    + {% endfor %} + {% endfor %} +
    + +
    + + +
    +
    +{% else %} +

    {{ _("There are no views created for this dataset yet.") }}

    +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/metadata.html b/ckanext/iaea/templates/package/metadata.html new file mode 100644 index 0000000..290289c --- /dev/null +++ b/ckanext/iaea/templates/package/metadata.html @@ -0,0 +1,6 @@ +{% extends "package/read_base.html" %} +{# Reuse of activity template for additonal info#} + +{% block primary_content_inner %} +{% snippet "package/snippets/additional_info.html", pkg_dict=pkg %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/read.html b/ckanext/iaea/templates/package/read.html new file mode 100644 index 0000000..c349485 --- /dev/null +++ b/ckanext/iaea/templates/package/read.html @@ -0,0 +1,11 @@ +{% ckan_extends %} + +{% block package_description %} +{% endblock %} + +{% block package_resources %} + {% snippet "package/snippets/resources_list.html", pkg=pkg, resources=pkg.resources %} +{% endblock %} + +{% block package_additional_info %} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/read_base.html b/ckanext/iaea/templates/package/read_base.html index d224d03..aee1a48 100644 --- a/ckanext/iaea/templates/package/read_base.html +++ b/ckanext/iaea/templates/package/read_base.html @@ -1,4 +1,160 @@ {% ckan_extends %} + +{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ super() }}{% endblock %} + +{% block head_extras -%} + {{ super() }} + {% set description = h.markdown_extract(pkg.notes, extract_length=200)|forceescape %} + + +{% endblock -%} + + + +{% block content_action %} + {% if h.check_access('package_update', {'id':pkg.id }) %} + {% link_for _('Manage'), controller='dataset', action='edit', id=pkg.name, class_='btn btn-default', icon='wrench' %} + {% endif %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name) }} + {{ h.build_nav_icon('iaea.metadata', _('Metadata'), id=pkg.name, icon='fa fa-tag') }} + {% if h.check_access('package_update', {'id':pkg.id }) %} + {{ h.build_nav_icon('iaea.feature_view', _('View'), id=pkg.name, icon='check-square-o') }} + {% endif %} +{% endblock %} + +{% block primary_content_inner %} + {% block package_revision_info %} + {% if c.revision_date %} +
    +

    + {% set timestamp = h.render_datetime(c.revision_date, with_hours=True) %} + {% set url = h.url_for(controller='dataset', action='read', id=pkg.name) %} + + {% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the current revision.{% endtrans %} +

    +
    + {% endif %} + {% endblock %} +{% endblock %} +{% block pre_primary %} +
    +
    +
    + {% if pkg.private %} + + + {{ _('Private') }} + + {% endif %} +

    + {% block page_heading %} + {{ h.dataset_display_name(pkg) }} + {% if pkg.state.startswith('draft') %} + [{{ _('Draft') }}] + {% endif %} + {% if pkg.state == 'deleted' %} + [{{ _('Deleted') }}] + {% endif %} + {% endblock %} +

    + {% block package_notes %} + {% if pkg.notes %} +
    + {{ h.render_markdown(h.get_translated(pkg, 'notes')) }} +
    + {% endif %} + {% endblock %} +
    +
    +
    + {% block title_and_actions_right_side %} + {% if pkg.organization %} + {% set org = h.get_organization(pkg.organization.name) %} + {% set url_org = h.url_for(pkg.organization.type + '_read', id=pkg.organization.name, ) %} +
    +
    + + {{ pkg.organization.name }} + +
    +
    + {% endif %} + {% endblock %} +
    +
    +
    +
    +
    + Last Update: + {{pkg.metadata_modified}} + +
    +
    + {% if pkg.maintainer_email %} +
    + + + Contact dataset maintainer + +
    + {% endif %} + +
    +
    +
    + {% if pkg.featured_view %} +
    +

    Featured Visualization

    +
    + + + Fullscreen + +
    +
    + + {% endif %} +{% endblock %} + + +{% block secondary_content %} +
    + {% block secondary_help_content %} +
    +
    +

    {{ _('Activity') }}

    +
    +
    +
    + {#{ h.package_activity_html(pkg.id) | safe }#} + {% endblock %} + + {% block package_organization %} + {% if pkg.organization %} + {% set org = h.get_organization(pkg.organization.id) %} + {% snippet "snippets/organization.html", organization=org, has_context_title=true %} + {% endif %} + {% endblock %} + + + {% block package_info %} + + {% endblock %} + + {% block package_license %} + {% endblock %} +
    +{% endblock %} + {% block main_content %} {{ super() }}
    diff --git a/ckanext/iaea/templates/package/snippets/additional_info.html b/ckanext/iaea/templates/package/snippets/additional_info.html new file mode 100644 index 0000000..cf33483 --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/additional_info.html @@ -0,0 +1,90 @@ +
    + + + + + + + + + {% block package_additional_info %} + {% if pkg_dict.url %} + + + {% if h.is_url(pkg_dict.url) %} + + {% else %} + + {% endif %} + + {% endif %} + + {% if pkg_dict.author_email %} + + + + + {% elif pkg_dict.author %} + + + + + {% endif %} + + {% if pkg_dict.maintainer_email %} + + + + + {% elif pkg_dict.maintainer %} + + + + + {% endif %} + + {% if pkg_dict.version %} + + + + + {% endif %} + + {% if h.check_access('package_update',{'id':pkg_dict.id}) %} + + + + + {% endif %} + {% if pkg_dict.metadata_modified %} + + + + + {% endif %} + {% if pkg_dict.metadata_created %} + + + + + + {% endif %} + + {% block extras scoped %} + {% for extra in h.sorted_extras(pkg_dict.extras) %} + {% set key, value = extra %} + + + + + {% endfor %} + {% endblock %} + + {% endblock %} + +
    {{ _('Field') }}{{ _('Value') }}
    {{ _('Source') }}{{ h.link_to(pkg_dict.url, pkg_dict.url, rel='foaf:homepage', target='_blank') }}{{ pkg_dict.url }}
    {{ _("Author") }}{{ h.mail_to(email_address=pkg_dict.author_email, name=pkg_dict.author) }}
    {{ _("Author") }}{{ pkg_dict.author }}
    {{ _('Maintainer') }}{{ h.mail_to(email_address=pkg_dict.maintainer_email, name=pkg_dict.maintainer) }}
    {{ _('Maintainer') }}{{ pkg_dict.maintainer }}
    {{ _("Version") }}{{ pkg_dict.version }}
    {{ _("State") }}{{ _(pkg_dict.state) }}
    {{ _("Last Updated") }} + {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_modified %} +
    {{ _("Created") }} + {% snippet 'snippets/local_friendly_datetime.html', datetime_obj=pkg_dict.metadata_created %} +
    {{ _(key) }}{{ value }}
    +
    \ No newline at end of file diff --git a/ckanext/iaea/templates/package/snippets/view_form.html b/ckanext/iaea/templates/package/snippets/view_form.html new file mode 100644 index 0000000..6f6af3d --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/view_form.html @@ -0,0 +1,9 @@ +{% ckan_extends %} + +{% block view_form_filters %} + {% if h.resource_view_is_filterable(resource_view) %} + {% snippet 'package/snippets/view_form_filters.html', resource=resource, resource_view=resource_view %} + {% endif %} + {% snippet 'package/snippets/view_form_field_filter.html', resource=resource, resource_view=resource_view, data=data, errors=errors %} + {# resource 'iaea/view_js' #} +{% endblock %} \ No newline at end of file diff --git a/ckanext/iaea/templates/package/snippets/view_form_field_filter.html b/ckanext/iaea/templates/package/snippets/view_form_field_filter.html new file mode 100644 index 0000000..4e815aa --- /dev/null +++ b/ckanext/iaea/templates/package/snippets/view_form_field_filter.html @@ -0,0 +1,22 @@ +{% import 'macros/form.html' as form %} +{% macro suggestedFilter(name, fields, suggested_filter_fields, label = '', error = '', options = '', classes = [], is_required = false) %} + {% set classes = (classes | list) %} + {%- set extra_html = caller() if caller -%} + {% call form.input_block(id or name, label or name, error, classes, extra_html = extra_html, is_required = is_required) %} +
    + +
    + {% endcall %} +{% endmacro %} + +{% set fields = h.resource_view_get_fields(resource) %} +
    + {{ + suggestedFilter('Suggested filter fields', label=_('Suggested filter fields'), fields=fields, suggested_filter_fields=data.suggested_filter_fields, error=errors, classes=[""]) + }} +
    \ No newline at end of file diff --git a/ckanext/iaea/view.py b/ckanext/iaea/view.py new file mode 100644 index 0000000..cea03ca --- /dev/null +++ b/ckanext/iaea/view.py @@ -0,0 +1,197 @@ +import logging +import ckan.logic as logic +import ckan.lib.base as base +import ckan.model as model +from flask.views import MethodView +import ckan.lib.dictization.model_dictize as model_dictize +from itertools import groupby +import ckan.lib.helpers as h + +from ckan.common import _, c, request + +log = logging.getLogger(__name__) + +render = base.render +abort = base.abort + +NotFound = logic.NotFound +NotAuthorized = logic.NotAuthorized +ValidationError = logic.ValidationError +check_access = logic.check_access +get_action = logic.get_action + + +def metadata(id): # noqa + """Render this package's public activity stream page. + """ + context = { + u'model': model, + u'session': model.Session, + u'user': c.user, + u'for_view': True, + u'auth_user_obj': c.userobj + } + + data_dict = {u'id': id} + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + # TODO: remove + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/metadata.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + } + ) + + +class FeatureView(MethodView): + """ feature selected view on dataset page + """ + + def _prepare(self, id): + context = { + u'model': model, + u'session': model.Session, + u'user': c.user, + u'auth_user_obj': c.userobj, + u'save': u'save' in request.form + } + try: + check_access(u'package_update', context, {u'id': id}) + + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort( + 403, + _(u'Unauthorized to edit %s') % u'' + ) + + return context + + def post(self, id): + context = self._prepare(id) + + featured_view = request.form.get('featured_view', False) + data_dict = {u'id': id} + if request.form['submit'] == 'submit': + if not featured_view: + h.flash_error('Please select view from the list.') + data_dict['featured_view'] = '' + else: + data_dict['featured_view'] = featured_view + else: + data_dict['featured_view'] = '' + + # update package with selected featured view + try: + pkg_dict = get_action(u'package_patch')(context, data_dict) + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + package_views = model.Session.query( + model.ResourceView + ).join(model.Resource).filter(model.Resource.package_id == pkg_dict['id']).all() + + package_views_list = model_dictize.resource_view_list_dictize( + package_views, context) + + package_views_dict = [] + package_views_list = sorted(package_views_list, key=lambda k: k['resource_id']) + for k, v in groupby(package_views_list, key=lambda x: x['resource_id']): + [resource_name, state ] = model.Session.query(model.Resource.name.label('name'), model.Resource.state.label('state')).filter( + model.Resource.id == k).first() + + if state == 'deleted': + continue + else: + view_dict = { + 'resource_name': resource_name, + 'resource_id': k, + 'views': list(v) + } + + package_views_dict.append(view_dict) + + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/features_view.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + u'package_views': package_views_dict + }) + + def get(self, id): + context = self._prepare(id) + data_dict = {u'id': id} + try: + pkg_dict = get_action(u'package_show')(context, data_dict) + pkg = context[u'package'] + dataset_type = pkg_dict[u'type'] or u'dataset' + except NotFound: + return base.abort(404, _(u'Dataset not found')) + except NotAuthorized: + return base.abort(403, _(u'Unauthorized to read dataset %s') % id) + + package_views = model.Session.query( + model.ResourceView + ).join(model.Resource).filter(model.Resource.package_id == pkg_dict['id']).all() + + package_views_list = model_dictize.resource_view_list_dictize( + package_views, context) + + package_views_dict = [] + package_views_list = sorted(package_views_list, key=lambda k: k['resource_id']) + for k, v in groupby(package_views_list, key=lambda x: x['resource_id']): + [resource_name, state ] = model.Session.query(model.Resource.name.label('name'), model.Resource.state.label('state')).filter( + model.Resource.id == k).first() + + if state == 'deleted': + continue + else: + view_dict = { + 'resource_name': resource_name, + 'resource_id': k, + 'views': list(v) + } + + package_views_dict.append(view_dict) + + c.pkg_dict = pkg_dict + c.pkg = pkg + + return base.render( + u'package/features_view.html', { + u'dataset_type': dataset_type, + u'pkg_dict': pkg_dict, + u'pkg': pkg, + u'id': id, # i.e. package's current name + u'package_views': package_views_dict + }) \ No newline at end of file