Move NTB related code from planning component (#344)

* Add NTBEventXMLFeedParser with test cases.

* Add NTB related feeding service test cases.

* NTB Event formatter.

* Behave tests for NTB events.

* Formatter and file feed service test cases.

* Update NTBEventFormatter with latest changes.

* Flake8.

* Separate parser implementation from feeding service.

* SDNTB-604 Typo in test feature name.

server/features/environment.py

@@ -51,6 +51,31 @@ def setup_ntb_event_api_provider(context):
         set_placeholder(context, 'PROVIDER_ID', str(result[0]))
 
 
+def setup_ntb_event_file_provider(context):
+    app = context.app
+    context.providers = {}
+    context.ingest_items = ingest_items
+    path_to_fixtures = os.path.join(
+        os.path.abspath(os.path.dirname(ntb.__file__)), 'tests', 'io', 'fixtures', 'ntb_events_file'
+    )
+    providers = [
+        {
+            'name': 'ntb-events-file',
+            'source': 'ntb',
+            'feeding_service': 'event_file',
+            'feed_parser': 'ntb_event_xml',
+            'is_closed': False,
+            'config': {
+                'path': path_to_fixtures
+            }
+        }
+    ]
+
+    with app.test_request_context(app.config['URL_PREFIX']):
+        result = superdesk.get_resource_service('ingest_providers').post(providers)
+        context.providers['ntb'] = result[0]
+
+
 def setup_ntb_vocabulary(context):
     with context.app.app_context():
         # prepopulate vocabularies
@@ -91,3 +116,6 @@ def before_scenario(context, scenario):
 
         if 'ntb_vocabulary' in scenario.tags:
             setup_ntb_vocabulary(context)
+
+        if 'events_ingest' in scenario.tags:
+            setup_ntb_event_file_provider(context)

server/features/ingest_events.feature → server/features/ingest_ntb_events_api.feature

@@ -1,4 +1,4 @@
-Feature: Events ingest
+Feature: Ingest NTB Events api
 
     @ntb_vocabulary @auth @ntb_event_api_provider
     Scenario: Ingest NTB event API xml

server/features/ingest_ntb_events_file.feature

@@ -0,0 +1,105 @@
+Feature: Events Ingest
+    @auth @events_ingest
+    Scenario: Ingest NTB Event file
+        Given empty "events"
+        When we get "/events"
+        Then we get list with 0 items
+        When we fetch events from "ntb-events-file" ingest "ntb_event_xml"
+        When we get "/events"
+        Then we get list with 1 items
+        """
+            {
+                "_items": [{
+                    "_id": "NTB-123456",
+                    "original_creator": "__no_value__"
+                }]
+            }
+        """
+
+    @auth @events_ingest
+    Scenario: Duplicate Ingested NTB Event file
+        Given empty "events"
+        When we get "/events"
+        Then we get list with 0 items
+        When we fetch events from "ntb-events-file" ingest "ntb_event_xml"
+        When we get "/events"
+        Then we get list with 1 items
+        """
+            {
+                "_items": [{
+                    "_id": "NTB-123456",
+                    "original_creator": "__no_value__"
+                }]
+            }
+        """
+        When we duplicate event "NTB-123456"
+        Then  we get OK response
+        When we get "/events"
+        Then we get list with 2 items
+        """
+            {
+                "_items": [
+                    {
+                        "_id": "NTB-123456",
+                        "original_creator": "__no_value__",
+                        "name": "Original Content",
+                        "definition_short": "Original Content",
+                        "state": "ingested"
+                    },
+                    {
+                        "_id": "#DUPLICATE_EVENT_ID#",
+                        "original_creator": "#CONTEXT_USER_ID#",
+                        "name": "duplicate",
+                        "definition_short": "duplicate",
+                        "duplicate_from": "NTB-123456",
+                        "state": "draft"
+                    }
+                ]
+            }
+        """
+
+    @auth @events_ingest
+    Scenario: We reschedule ingested Ingest NTB Event
+        Given empty "events"
+        When we get "/events"
+        Then we get list with 0 items
+        When we fetch events from "ntb-events-file" ingest "ntb_event_xml"
+        When we get "/events"
+        Then we get list with 1 items
+        """
+            {
+                "_items": [{
+                    "_id": "NTB-123456",
+                    "original_creator": "__no_value__",
+                    "state": "ingested"
+                }]
+            }
+        """
+        When we post to "/events/NTB-123456/lock"
+        """
+        {"lock_action": "reschedule"}
+        """
+        Then we get OK response
+        When we perform reschedule on events "NTB-123456"
+        """
+        {
+            "reason": "Changed to the next day!",
+            "dates": {
+                "start": "2029-11-22T12:00:00.000Z",
+                "end": "2029-11-22T14:00:00.000Z",
+                "tz": "Australia/Sydney"
+            }
+        }
+        """
+        Then we get OK response
+        When we get "/events"
+        Then we get list with 1 items
+        """
+            {
+                "_items": [{
+                    "_id": "NTB-123456",
+                    "original_creator": "__no_value__",
+                    "state": "draft"
+                }]
+            }
+        """

server/features/steps/steps.py

@@ -9,13 +9,22 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 import os
+from copy import deepcopy
 from unittest import mock
-from superdesk import get_resource_service
+from datetime import timedelta
+
+from flask import json
+from superdesk import get_resource_service, etree
+from superdesk.utc import utcnow
+from superdesk.io import get_feeding_service
+from superdesk.io.commands.update_ingest import LAST_ITEM_UPDATE
 from superdesk.tests import set_placeholder
 from superdesk.tests.steps import (
-    when, then, get_json_data
+    when, then, get_json_data, post_data, apply_placeholders,
+    get_prefixed_url, get_res, if_match
 )
 from superdesk.io.commands import update_ingest
+from superdesk.io.feed_parsers import XMLFeedParser
 from superdesk.io.feeding_services import http_base_service
 
 
@@ -82,3 +91,76 @@ def step_impl_then_get_list_without_ntb_id(context):
 def step_impl_then_save_event_id(context):
     items = get_json_data(context.response)['_items']
     set_placeholder(context, 'EVENT_TO_PATCH', str(items[0]['guid']))
+
+
+@when('we fetch events from "{provider_name}" ingest "{guid}"')
+def step_impl_fetch_from_provider_ingest(context, provider_name, guid):
+    with context.app.test_request_context(context.app.config['URL_PREFIX']):
+        ingest_provider_service = get_resource_service('ingest_providers')
+        provider = ingest_provider_service.find_one(name=provider_name, req=None)
+        provider_service = get_feeding_service(provider['feeding_service'])
+        file_path = os.path.join(provider.get('config', {}).get('path', ''), guid)
+        feeding_parser = provider_service.get_feed_parser(provider)
+        if isinstance(feeding_parser, XMLFeedParser):
+            with open(file_path, 'rb') as f:
+                xml_string = etree.etree.fromstring(f.read())
+                parsed = feeding_parser.parse(xml_string, provider)
+        else:
+            parsed = feeding_parser.parse(file_path, provider)
+
+        items = [parsed] if not isinstance(parsed, list) else parsed
+
+        for item in items:
+            item['versioncreated'] = utcnow()
+            item['expiry'] = utcnow() + timedelta(minutes=20)
+
+        failed = context.ingest_items(items, provider, provider_service)
+        assert len(failed) == 0, failed
+
+        provider = ingest_provider_service.find_one(name=provider_name, req=None)
+        ingest_provider_service.system_update(provider['_id'], {LAST_ITEM_UPDATE: utcnow()}, provider)
+
+        for item in items:
+            set_placeholder(context, '{}.{}'.format(provider_name, item['guid']), item['_id'])
+
+
+@when('we duplicate event "{event_id}"')
+def step_impl_when_we_duplicate_event(context, event_id):
+    with context.app.test_request_context(context.app.config['URL_PREFIX']):
+        events_service = get_resource_service('events')
+        original_event = events_service.find_one(req=None, _id=event_id)
+        duplicate_event = deepcopy(original_event)
+
+        for key, value in original_event.items():
+            if key.startswith('_'):
+                duplicate_event.pop(key, None)
+
+        for key in ['state', 'firstcreated', 'versioncreated', 'ingest_provider', 'guid']:
+            duplicate_event.pop(key, None)
+
+        duplicate_event['duplicate_from'] = event_id
+        duplicate_event['dates']['start'] = "2099-01-02"
+        duplicate_event['dates']['end'] = "2099-01-03"
+        duplicate_event['unique_id'] = 456
+        duplicate_event['definition_short'] = 'duplicate'
+        duplicate_event['name'] = 'duplicate'
+
+        context.text = json.dumps(duplicate_event)
+        item = post_data(context, '/events')
+        set_placeholder(context, 'DUPLICATE_EVENT_ID', item['_id'])
+
+
+@when('we perform {action} on {resource} "{item_id}"')
+def step_imp_when_action_resource(context, action, resource, item_id):
+    data = context.text or {}
+    resource = apply_placeholders(context, resource)
+    item_id = apply_placeholders(context, item_id)
+
+    item_url = '/{}/{}'.format(resource, item_id)
+    action_url = '/{}/{}/{}'.format(resource, action, item_id)
+
+    res = get_res(item_url, context)
+    headers = if_match(context, res.get('_etag'))
+
+    context.response = context.client.patch(get_prefixed_url(context.app, action_url),
+                                            data=json.dumps(data), headers=headers)

server/ntb/io/feed_parsers/__init__.py

@@ -10,3 +10,4 @@
 from . import ntb_events_api_xml # noqa
 from . import solita # noqa
 from . import solita_se # noqa
+from . import ntb_event_xml # noqa

server/ntb/io/feed_parsers/ntb_event_xml.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8; -*-
+#
+# This file is part of Superdesk.
+#
+# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
+#
+# For the full copyright and license information, please see the
+# AUTHORS and LICENSE files distributed with this source code, or
+# at https://www.sourcefabric.org/superdesk/license
+
+import logging
+import xml.etree.ElementTree as ET
+
+from superdesk.errors import ParserError
+from superdesk.io.feed_parsers import XMLFeedParser
+from superdesk.io.registry import register_feed_parser
+from superdesk.metadata.item import ITEM_TYPE, CONTENT_TYPE, GUID_FIELD, GUID_NEWSML, FORMAT, FORMATS
+from superdesk.metadata.utils import generate_guid
+from superdesk.utc import utcnow
+
+logger = logging.getLogger(__name__)
+
+
+class NTBEventXMLFeedParser(XMLFeedParser):
+    """NTB Event XML parser.
+
+    Feed Parser which can parse an NTB created XML file exported from Outlook
+    the firstcreated and versioncreated times are localised.
+    """
+
+    NAME = 'ntb_event_xml'
+
+    label = 'NTB Event XML'
+
+    def can_parse(self, xml):
+        return xml.tag.endswith('document')
+
+    def parse_email(self, content, content_type, provider):
+        if content_type != 'text/xml':
+            raise ParserError.parseMessageError('Not supported content type.')
+
+        content.seek(0)
+        xml = ET.parse(content)
+        return self.parse(xml.getroot(), provider)
+
+    def parse_file(self, fstream, provider):
+        xml = ET.parse(fstream)
+        return self.parse(xml.getroot(), provider)
+
+    def parse_http(self, content, provider):
+        xml = ET.fromstring(content)
+        return self.parse(xml, provider)
+
+    def parse(self, xml, provider=None, content=None):
+        items = []
+        try:
+            # parse xml file, only expecting one event per file
+            if not ET.iselement(xml.find('guid')):
+                guid = generate_guid(type=GUID_NEWSML)
+            else:
+                guid = xml.find('guid').text
+
+            item = {
+                ITEM_TYPE: CONTENT_TYPE.EVENT,
+                GUID_FIELD: guid,
+                FORMAT: FORMATS.PRESERVED
+            }
+            item['name'] = xml.find('title').text
+            item['definition_short'] = xml.find('title').text
+            item['definition_long'] = xml.find('content').text
+            item['dates'] = {
+                'start': xml.find('timeStart').text,
+                'end': xml.find('timeEnd').text,
+                'tz': ''
+            }
+            # add location
+            item['location'] = [{
+                'name': xml.find('location').text,
+                'qcode': '',
+                'geo': ''
+            }]
+            if ET.iselement(xml.find('geo')):
+                geo = xml.find('geo')
+                item['location'][0]['geo'] = '%s, %s' % (geo.find('latitude').text, geo.find('longitude').text)
+            # IMPORTANT: firstcreated must be less than 2 days past
+            # we must preserve the original event created and updated in some other fields
+            item['firstcreated'] = utcnow()
+            item['versioncreated'] = utcnow()
+            items.append(item)
+
+            return items
+        except Exception as ex:
+            raise ParserError.parseMessageError(ex, provider)
+
+
+register_feed_parser(NTBEventXMLFeedParser.NAME, NTBEventXMLFeedParser())

server/ntb/io/feed_parsers/ntb_nifs.py

@@ -34,6 +34,7 @@ class NTBNIFSFeedParser(FeedParser):
     Feed Parser which can parse an NTB created XML file exported from Outlook
     the firstcreated and versioncreated times are localised.
     """
+
     NAME = 'ntb_nifs'
     label = 'NIFS Sport Events'
     subjects_map = None

server/ntb/io/feed_parsers/ritzau.py

@@ -18,6 +18,7 @@ class RitzauFeedParser(RitzauFeedParser):
     """
     Feed Parser which can parse Ritzau XML feed
     """
+
     _subjects_map = None
 
     NAME = 'ntb_ritzau'

server/ntb/macros/en_to_no_macro.py

@@ -1,9 +1,9 @@
 """
-    En to NO Metadata Macro will perform the following changes to current content item:
-    - change the byline to "(NPK-NTB)"
-    - change the signoff to "npk@npk.no"
-    - change the body footer to "(©NPK)" - NB: copyrightsign, not @
-    - change the service to "NPKSisteNytt"
+En to NO Metadata Macro will perform the following changes to current content item:
+- change the byline to "(NPK-NTB)"
+- change the signoff to "npk@npk.no"
+- change the body footer to "(©NPK)" - NB: copyrightsign, not @
+- change the service to "NPKSisteNytt"
 """
 
 

server/ntb/macros/extract_html.py

@@ -14,7 +14,7 @@
 
 def extract_html_macro(item, **kwargs):
     """
-        Delete from body_html all html tags except links
+    Delete from body_html all html tags except links
     """
     if 'body_html' not in item:
         return None