SDNTB-620 A duplicate published in NTB NITF format has the same NTBID…

… as the original story (#362) * Import mock from unittest module. * Flake8 bugbear plugin exceptions. * SDNTB-620 [NITF] Use original item guid/_id for NTBID when update is published. * SDNTB-620 [MACROS] Use load_macros helper to load NTB macros. * SDNTB-620 [MACROS] Make a translation link between items. * Use develop branch for superdesk-core. * Remove chrome from travis. * Fix typo.
superdesk · Jan 17, 2020 · b536606 · b536606
1 parent af71215
commit b536606
Show file tree

Hide file tree

Showing 9 changed files with 179 additions and 46 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,11 +10,9 @@ addons:
   apt:
     sources:
     - mongodb-3.0-precise
-    - google-chrome
     - elasticsearch-2.x
     packages:
     - mongodb-org-server
-    - google-chrome-stable
     - elasticsearch
 
 cache:

diff --git a/server/ntb/macros/__init__.py b/server/ntb/macros/__init__.py
@@ -5,24 +5,9 @@
 Use `superdesk.macro_register.macros.register` for registration.
 """
 
-import os
-import sys
-import imp
-import importlib
+import os.path
+from superdesk.macros import load_macros
 
-macro_replacement_fields = {'body_html', 'body_text', 'abstract', 'headline', 'slugline', 'description_text'}
-macros_folder = os.path.realpath(os.path.dirname(__file__))
-
-macros = [f[:-3] for f in os.listdir(macros_folder)
-          if f.endswith('.py') and not f.endswith('_test.py') and not f.startswith('__')]
 
-for macro in macros:
-    try:
-        module = 'ntb.macros.{}'.format(macro)
-        if module in sys.modules.keys():
-            m = sys.modules[module]
-            imp.reload(m)
-        else:
-            importlib.import_module(module)
-    except Exception:
-        pass
+macros_folder = os.path.realpath(os.path.dirname(__file__))
+load_macros(macros_folder, package_prefix="ntb.macros")
diff --git a/server/ntb/macros/npk_metadata.py b/server/ntb/macros/npk_metadata.py
@@ -1,17 +1,38 @@
-"""
-NPKSisteNytt Metadata Macro will perform the following changes to current content item:
-- change the byline to "(NPK-NTB)"
-- change the signoff to "npk@npk.no"
-- change the body footer to "(©NPK)" - NB: copyrightsign, not @
-- change the service to "NPKSisteNytt"
-"""
+import logging
+
+from superdesk import get_resource_service
+from superdesk.errors import StopDuplication
+
+logger = logging.getLogger(__name__)
 
 
 def npk_metadata_macro(item, **kwargs):
+    """
+    The NPKSisteNytt Metadata Macro will perform the following changes to current content item:
+    - change the byline to "(NPK-NTB)"
+    - change the signoff to "npk@npk.no"
+    - change the body footer to "(©NPK)" - NB: copyrightsign, not @
+    - change the service to "NPKSisteNytt"
+    - change the language to "nn-NO"
+    """
+
+    desk_id = kwargs.get('dest_desk_id')
+    if not desk_id:
+        logger.warning("no destination id specified")
+        return
+    stage_id = kwargs.get('dest_stage_id')
+    if not stage_id:
+        logger.warning("no stage id specified")
+        return
+
+    archive_service = get_resource_service('archive')
+    move_service = get_resource_service('move')
+    translate_service = get_resource_service('translate')
+
+    # change item
     item['byline'] = 'NPK-' + item.get('byline', '')
     item['sign_off'] = 'npk@npk.no'
     item['body_footer'] = '(©NPK)'
-    item['language'] = 'nn-NO'
     item['anpa_category'] = [
         {
             'qcode': 's',
@@ -21,7 +42,17 @@ def npk_metadata_macro(item, **kwargs):
             'scheme': None
         }
     ]
-    return item
+    item['language'] = 'nn-NO'
+    # make a translation
+    new_id = translate_service.create([item])
+    # move item to the desk/stage
+    dest = {"task": {"desk": desk_id, "stage": stage_id}}
+    move_service.move_content(new_id, dest)
+    # apply the update
+    new_item = archive_service.find_one(req=None, _id=new_id)
+    archive_service.put(new_id, new_item)
+
+    raise StopDuplication
 
 
 name = 'NPKSisteNytt Metadata Macro'

diff --git a/server/ntb/publish/ntb_nitf.py b/server/ntb/publish/ntb_nitf.py
@@ -167,8 +167,8 @@ def _format_docdata_dateissue(self, article, docdata):
             attrib={'norm': article['versioncreated'].astimezone(tz).strftime("%Y-%m-%dT%H:%M:%S")})
 
     def _format_docdata_doc_id(self, article, docdata):
-        doc_id = "NTB{family_id}_{version:02}".format(
-            family_id=article['family_id'],
+        doc_id = "{ntbid}_{version:02}".format(
+            ntbid=self._get_ntbid(article),
             version=_get_rewrite_sequence(article))
         etree.SubElement(docdata, 'doc-id', attrib={'regsrc': 'NTB', 'id-string': doc_id})
 
@@ -257,7 +257,7 @@ def _format_meta(self, article, head, destination, pub_seq_num):
             etree.SubElement(head, 'meta', {'name': 'NTBStikkord', 'content': self._get_ntb_slugline(article)})
         etree.SubElement(head, 'meta', {'name': 'subject', 'content': self._get_ntb_subject(article)})
 
-        etree.SubElement(head, 'meta', {'name': 'NTBID', 'content': 'NTB{}'.format(article['family_id'])})
+        etree.SubElement(head, 'meta', {'name': 'NTBID', 'content': self._get_ntbid(article)})
 
         # these static values never change
         etree.SubElement(head, 'meta', {'name': 'NTBDistribusjonsKode', 'content': 'ALL'})
@@ -344,6 +344,7 @@ def _format_body_content(self, article, body_content):
 
         # media
         media_data = []
+        associations = {}
         try:
             associations = article['associations']
         except KeyError:
@@ -498,6 +499,48 @@ def _get_original_href(self, data):
             except (StopIteration, KeyError):
                 logger.warning("Can't find source for media {}".format(data.get('guid', '')))
 
+    def _get_ntbid(self, article):
+        """
+        Return NTBID.
+        If article is an update, use original item guid (if no guid use _id) to create an NTBID.
+        If article is not an update use guid/_id of a current article to to create an NTBID.
+        :param article: article
+        :return: NTBID
+        """
+
+        if 'rewrite_of' in article:
+            # pymongo is used to have an ability to specify a projection
+            # it's not possible to specify a projection while using eve's resource based service
+            # https://github.com/pyeve/eve/blob/master/eve/io/base.py#L449
+            db_archive = app.data.mongo.pymongo('archive').db['archive']
+            items = tuple(db_archive.find(
+                {
+                    'family_id': article['family_id'],
+                    '_id': {
+                        '$ne': article['_id']
+                    }
+                },
+                {'rewrite_of': 1, 'guid': 1}
+            ))
+            items_dict = {i['_id']: i for i in items}
+            # find original item id
+            orig_id = article['rewrite_of']
+            while True:
+                try:
+                    # if no rewrite_of, it means that item is original
+                    orig_id = items_dict[orig_id]['rewrite_of']
+                except KeyError:
+                    break
+            # try to get guid
+            try:
+                orig_id = items_dict[orig_id]['guid']
+            except KeyError:
+                pass
+
+            return 'NTB{}'.format(orig_id)
+
+        return 'NTB{}'.format(article.get('guid', article['_id']))
+
     def _format_body_end(self, article, body_end):
         try:
             emails = [s.strip() for s in article['sign_off'].split('/')]

diff --git a/server/ntb/tests/io/feeding_services/event_email_service_tests.py b/server/ntb/tests/io/feeding_services/event_email_service_tests.py
@@ -1,5 +1,5 @@
 import imaplib
-from mock import Mock, patch
+from unittest.mock import Mock, patch
 from planning.feeding_services.event_email_service import EventEmailFeedingService
 from planning.tests import TestCase
 from ntb.io.feed_parsers.ntb_event_xml import NTBEventXMLFeedParser

diff --git a/server/ntb/tests/publish/ntb_nitf_legacy_test.py b/server/ntb/tests/publish/ntb_nitf_legacy_test.py
@@ -56,11 +56,11 @@ def test_slugline(self):
     def test_doc_id(self):
         doc_id = self.nitf_xmls[0].find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ITEM_ID, 1))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ARTICLE['guid'], 1))
 
         doc_id = self.nitf_xmls[1].find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ITEM_ID, 1))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ARTICLE['guid'], 1))
 
     def test_filename(self):
         filename = self.nitf_xmls[0].find('head/meta[@name="filename"]')

diff --git a/server/ntb/tests/publish/ntb_nitf_test.py b/server/ntb/tests/publish/ntb_nitf_test.py
@@ -54,6 +54,7 @@
     'type': 'text',
     'priority': '2',
     '_id': 'urn:localhost.abc',
+    'guid': 'urn:localhost.guid.abc',
     'item_id': ITEM_ID,
     'family_id': ITEM_ID,
     # we use non latin1 chars in slugline to test encoding
@@ -244,7 +245,7 @@ def test_slugline(self):
     def test_doc_id(self):
         doc_id = self.nitf_xml.find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ITEM_ID, 1))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ARTICLE['guid'], 1))
 
     def test_pubdata(self):
         pubdata = self.nitf_xml.find('head/pubdata')
@@ -548,7 +549,7 @@ def test_meta(self):
         kanal = head.find('meta[@name="NTBKanal"]')
         self.assertEqual(kanal.get('content'), 'A')
         ntb_id = head.find('meta[@name="NTBID"]')
-        self.assertEqual(ntb_id.get('content'), 'NTB' + ITEM_ID)
+        self.assertEqual(ntb_id.get('content'), 'NTB' + ARTICLE['guid'])
         ntb_kilde = head.find('meta[@name="NTBKilde"]')
         self.assertEqual(ntb_kilde.get('content'), 'test ntb_pub_name')
         # priority
@@ -557,24 +558,22 @@ def test_meta(self):
 
     @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
     def test_update_id(self):
-        """Check use of family_id on update
+        """Check use of guid on update
 
         when family id is different from item_id (i.e. on updated item),
         family_id should be used for doc-id and ntbid
         """
         article = copy.deepcopy(self.article)
-        family_id = "test_family_id"
-        article['family_id'] = family_id
         article['rewrite_sequence'] = 3
         formatter_output = self.formatter.format(article, {'name': 'Test NTBNITF'})
         doc = formatter_output[0]['encoded_item']
         nitf_xml = etree.fromstring(doc)
         head = nitf_xml.find('head')
         ntb_id = head.find('meta[@name="NTBID"]')
-        self.assertEqual(ntb_id.get('content'), 'NTB' + family_id)
+        self.assertEqual(ntb_id.get('content'), 'NTB' + article['guid'])
         doc_id = nitf_xml.find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(family_id, 3))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(article['guid'], 3))
 
     @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
     def test_description_text_none(self):
@@ -620,10 +619,86 @@ def test_rewrite_sequence_none(self):
         doc = formatter_output[0]['encoded_item']
         nitf_xml = etree.fromstring(doc)
         doc_id = nitf_xml.find('head/docdata/doc-id')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(article['family_id'], 0))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(article['guid'], 0))
 
     @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
     def test_language_empty(self):
         article = copy.deepcopy(self.article)
         article.pop('language')
         self.formatter.format(article, {'name': 'Test NTBNITF'})
+
+
+class NTBNITFFormatterUpdateTest(TestCase):
+    archive = [
+        {
+            '_id': 'urn:localhost.abc',
+            'guid': 'urn:localhost.guid.abc',
+            'family_id': ITEM_ID,
+            'headline': 'test headline',
+            'abstract': TEST_ABSTRACT,
+            'body_html': TEST_BODY,
+            'type': 'text',
+            'priority': '2',
+            "slugline": "this is the slugline œ:?–",
+            'urgency': 2,
+            'versioncreated': NOW,
+            '_current_version': 5,
+            'version': 5,
+            'language': 'nb-NO'
+        },
+        {
+            '_id': 'urn:localhost.abc.up',
+            'guid': 'urn:localhost.guid.abc.up',
+            'family_id': ITEM_ID,
+            'rewrite_sequence': 1,
+            'rewrite_of': 'urn:localhost.abc',
+            'headline': 'test headline',
+            'abstract': TEST_ABSTRACT,
+            'body_html': TEST_BODY,
+            'type': 'text',
+            'priority': '2',
+            "slugline": "this is the slugline œ:?–",
+            'urgency': 2,
+            'versioncreated': NOW,
+            '_current_version': 5,
+            'version': 5,
+            'language': 'nb-NO'
+        },
+        {
+            '_id': 'urn:localhost.abc.up2',
+            'guid': 'urn:localhost.guid.abc.up2',
+            'family_id': ITEM_ID,
+            'rewrite_sequence': 2,
+            'rewrite_of': 'urn:localhost.abc.up',
+            'headline': 'test headline',
+            'abstract': TEST_ABSTRACT,
+            'body_html': TEST_BODY,
+            'type': 'text',
+            'priority': '2',
+            "slugline": "this is the slugline œ:?–",
+            'urgency': 2,
+            'versioncreated': NOW,
+            '_current_version': 5,
+            'version': 5,
+            'language': 'nb-NO'
+        }
+    ]
+
+    @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
+    def setUp(self):
+        init_app(self.app)
+        self.app.data.insert('archive', self.archive)
+        self.formatter = NTBNITFFormatter()
+        self.formatter_output = self.formatter.format(self.archive[-1], {'name': 'Test NTBNITF'})
+        self.doc = self.formatter_output[0]['encoded_item']
+        self.nitf_xml = etree.fromstring(self.doc)
+
+    def test_ntbid(self):
+        head = self.nitf_xml.find('head')
+        ntb_id = head.find('meta[@name="NTBID"]')
+        self.assertEqual(ntb_id.get('content'), 'NTB' + self.archive[0]['guid'])
+
+    def test_docid(self):
+        doc_id = self.nitf_xml.find('head/docdata/doc-id')
+        self.assertEqual(doc_id.get('regsrc'), 'NTB')
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(self.archive[0]['guid'], 2))
diff --git a/server/requirements.txt b/server/requirements.txt
@@ -2,6 +2,6 @@ gunicorn==19.7.1
 honcho==1.0.1
 newrelic==2.88.1.73
 feedgen==0.7.0
-superdesk-core==1.32.2
 
+-e git+git://github.com/superdesk/superdesk-core.git@develop#egg=Superdesk-Core
 -e git+git://github.com/superdesk/superdesk-planning.git@1.10.0#egg=superdesk-planning
diff --git a/server/setup.cfg b/server/setup.cfg
@@ -1,5 +1,6 @@
 [flake8]
 max-line-length=120
 exclude=env,bin,lib,include,src
-ignore=F811,D200,D202,D205,D400,D401,D100,D101,D102,D103,D104,D105,D107,W503,W504,W605,F401,E261,F841
+ignore=F811,D200,D202,D205,D400,D401,D100,D101,D102,D103,D104,D105,D107,W503,W504,W605,F401,E261,F841,
+       B010,B009,B007,B305,B011
 # W504, W605, F401, E261 and F841 are temporally ignored, due to recent changes in flake8