Revert "SDNTB-620 A duplicate published in NTB NITF format has the sa…

…me NTBID as the original story (#362)" This reverts commit b536606.
superdesk · May 15, 2020 · f972a1f · f972a1f
1 parent f564709
commit f972a1f
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 178 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,9 +10,11 @@ addons:
   apt:
     sources:
     - mongodb-3.0-precise
+    - google-chrome
     - elasticsearch-2.x
     packages:
     - mongodb-org-server
+    - google-chrome-stable
     - elasticsearch
 
 cache:

diff --git a/server/ntb/macros/__init__.py b/server/ntb/macros/__init__.py
@@ -5,9 +5,24 @@
 Use `superdesk.macro_register.macros.register` for registration.
 """
 
-import os.path
-from superdesk.macros import load_macros
-
+import os
+import sys
+import imp
+import importlib
 
+macro_replacement_fields = {'body_html', 'body_text', 'abstract', 'headline', 'slugline', 'description_text'}
 macros_folder = os.path.realpath(os.path.dirname(__file__))
-load_macros(macros_folder, package_prefix="ntb.macros")
+
+macros = [f[:-3] for f in os.listdir(macros_folder)
+          if f.endswith('.py') and not f.endswith('_test.py') and not f.startswith('__')]
+
+for macro in macros:
+    try:
+        module = 'ntb.macros.{}'.format(macro)
+        if module in sys.modules.keys():
+            m = sys.modules[module]
+            imp.reload(m)
+        else:
+            importlib.import_module(module)
+    except Exception:
+        pass
diff --git a/server/ntb/macros/npk_metadata.py b/server/ntb/macros/npk_metadata.py
@@ -1,38 +1,17 @@
-import logging
-
-from superdesk import get_resource_service
-from superdesk.errors import StopDuplication
-
-logger = logging.getLogger(__name__)
+"""
+NPKSisteNytt Metadata Macro will perform the following changes to current content item:
+- change the byline to "(NPK-NTB)"
+- change the signoff to "npk@npk.no"
+- change the body footer to "(©NPK)" - NB: copyrightsign, not @
+- change the service to "NPKSisteNytt"
+"""
 
 
 def npk_metadata_macro(item, **kwargs):
-    """
-    The NPKSisteNytt Metadata Macro will perform the following changes to current content item:
-    - change the byline to "(NPK-NTB)"
-    - change the signoff to "npk@npk.no"
-    - change the body footer to "(©NPK)" - NB: copyrightsign, not @
-    - change the service to "NPKSisteNytt"
-    - change the language to "nn-NO"
-    """
-
-    desk_id = kwargs.get('dest_desk_id')
-    if not desk_id:
-        logger.warning("no destination id specified")
-        return
-    stage_id = kwargs.get('dest_stage_id')
-    if not stage_id:
-        logger.warning("no stage id specified")
-        return
-
-    archive_service = get_resource_service('archive')
-    move_service = get_resource_service('move')
-    translate_service = get_resource_service('translate')
-
-    # change item
     item['byline'] = 'NPK-' + item.get('byline', '')
     item['sign_off'] = 'npk@npk.no'
     item['body_footer'] = '(©NPK)'
+    item['language'] = 'nn-NO'
     item['anpa_category'] = [
         {
             'qcode': 's',
@@ -42,17 +21,7 @@ def npk_metadata_macro(item, **kwargs):
             'scheme': None
         }
     ]
-    item['language'] = 'nn-NO'
-    # make a translation
-    new_id = translate_service.create([item])
-    # move item to the desk/stage
-    dest = {"task": {"desk": desk_id, "stage": stage_id}}
-    move_service.move_content(new_id, dest)
-    # apply the update
-    new_item = archive_service.find_one(req=None, _id=new_id)
-    archive_service.put(new_id, new_item)
-
-    raise StopDuplication
+    return item
 
 
 name = 'NPKSisteNytt Metadata Macro'

diff --git a/server/ntb/publish/ntb_nitf.py b/server/ntb/publish/ntb_nitf.py
@@ -167,8 +167,8 @@ def _format_docdata_dateissue(self, article, docdata):
             attrib={'norm': article['versioncreated'].astimezone(tz).strftime("%Y-%m-%dT%H:%M:%S")})
 
     def _format_docdata_doc_id(self, article, docdata):
-        doc_id = "{ntbid}_{version:02}".format(
-            ntbid=self._get_ntbid(article),
+        doc_id = "NTB{family_id}_{version:02}".format(
+            family_id=article['family_id'],
             version=_get_rewrite_sequence(article))
         etree.SubElement(docdata, 'doc-id', attrib={'regsrc': 'NTB', 'id-string': doc_id})
 
@@ -257,7 +257,7 @@ def _format_meta(self, article, head, destination, pub_seq_num):
             etree.SubElement(head, 'meta', {'name': 'NTBStikkord', 'content': self._get_ntb_slugline(article)})
         etree.SubElement(head, 'meta', {'name': 'subject', 'content': self._get_ntb_subject(article)})
 
-        etree.SubElement(head, 'meta', {'name': 'NTBID', 'content': self._get_ntbid(article)})
+        etree.SubElement(head, 'meta', {'name': 'NTBID', 'content': 'NTB{}'.format(article['family_id'])})
 
         # these static values never change
         etree.SubElement(head, 'meta', {'name': 'NTBDistribusjonsKode', 'content': 'ALL'})
@@ -344,7 +344,6 @@ def _format_body_content(self, article, body_content):
 
         # media
         media_data = []
-        associations = {}
         try:
             associations = article['associations']
         except KeyError:
@@ -499,48 +498,6 @@ def _get_original_href(self, data):
             except (StopIteration, KeyError):
                 logger.warning("Can't find source for media {}".format(data.get('guid', '')))
 
-    def _get_ntbid(self, article):
-        """
-        Return NTBID.
-        If article is an update, use original item guid (if no guid use _id) to create an NTBID.
-        If article is not an update use guid/_id of a current article to to create an NTBID.
-        :param article: article
-        :return: NTBID
-        """
-
-        if 'rewrite_of' in article:
-            # pymongo is used to have an ability to specify a projection
-            # it's not possible to specify a projection while using eve's resource based service
-            # https://github.com/pyeve/eve/blob/master/eve/io/base.py#L449
-            db_archive = app.data.mongo.pymongo('archive').db['archive']
-            items = tuple(db_archive.find(
-                {
-                    'family_id': article['family_id'],
-                    '_id': {
-                        '$ne': article['_id']
-                    }
-                },
-                {'rewrite_of': 1, 'guid': 1}
-            ))
-            items_dict = {i['_id']: i for i in items}
-            # find original item id
-            orig_id = article['rewrite_of']
-            while True:
-                try:
-                    # if no rewrite_of, it means that item is original
-                    orig_id = items_dict[orig_id]['rewrite_of']
-                except KeyError:
-                    break
-            # try to get guid
-            try:
-                orig_id = items_dict[orig_id]['guid']
-            except KeyError:
-                pass
-
-            return 'NTB{}'.format(orig_id)
-
-        return 'NTB{}'.format(article.get('guid', article['_id']))
-
     def _format_body_end(self, article, body_end):
         try:
             emails = [s.strip() for s in article['sign_off'].split('/')]

diff --git a/server/ntb/tests/io/feeding_services/event_email_service_tests.py b/server/ntb/tests/io/feeding_services/event_email_service_tests.py
@@ -1,5 +1,5 @@
 import imaplib
-from unittest.mock import Mock, patch
+from mock import Mock, patch
 from planning.feeding_services.event_email_service import EventEmailFeedingService
 from planning.tests import TestCase
 from ntb.io.feed_parsers.ntb_event_xml import NTBEventXMLFeedParser

diff --git a/server/ntb/tests/publish/ntb_nitf_legacy_test.py b/server/ntb/tests/publish/ntb_nitf_legacy_test.py
@@ -56,11 +56,11 @@ def test_slugline(self):
     def test_doc_id(self):
         doc_id = self.nitf_xmls[0].find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ARTICLE['guid'], 1))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ITEM_ID, 1))
 
         doc_id = self.nitf_xmls[1].find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ARTICLE['guid'], 1))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ITEM_ID, 1))
 
     def test_filename(self):
         filename = self.nitf_xmls[0].find('head/meta[@name="filename"]')

diff --git a/server/ntb/tests/publish/ntb_nitf_test.py b/server/ntb/tests/publish/ntb_nitf_test.py
@@ -54,7 +54,6 @@
     'type': 'text',
     'priority': '2',
     '_id': 'urn:localhost.abc',
-    'guid': 'urn:localhost.guid.abc',
     'item_id': ITEM_ID,
     'family_id': ITEM_ID,
     # we use non latin1 chars in slugline to test encoding
@@ -245,7 +244,7 @@ def test_slugline(self):
     def test_doc_id(self):
         doc_id = self.nitf_xml.find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ARTICLE['guid'], 1))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(ITEM_ID, 1))
 
     def test_pubdata(self):
         pubdata = self.nitf_xml.find('head/pubdata')
@@ -549,7 +548,7 @@ def test_meta(self):
         kanal = head.find('meta[@name="NTBKanal"]')
         self.assertEqual(kanal.get('content'), 'A')
         ntb_id = head.find('meta[@name="NTBID"]')
-        self.assertEqual(ntb_id.get('content'), 'NTB' + ARTICLE['guid'])
+        self.assertEqual(ntb_id.get('content'), 'NTB' + ITEM_ID)
         ntb_kilde = head.find('meta[@name="NTBKilde"]')
         self.assertEqual(ntb_kilde.get('content'), 'test ntb_pub_name')
         # priority
@@ -558,22 +557,24 @@ def test_meta(self):
 
     @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
     def test_update_id(self):
-        """Check use of guid on update
+        """Check use of family_id on update
 
         when family id is different from item_id (i.e. on updated item),
         family_id should be used for doc-id and ntbid
         """
         article = copy.deepcopy(self.article)
+        family_id = "test_family_id"
+        article['family_id'] = family_id
         article['rewrite_sequence'] = 3
         formatter_output = self.formatter.format(article, {'name': 'Test NTBNITF'})
         doc = formatter_output[0]['encoded_item']
         nitf_xml = etree.fromstring(doc)
         head = nitf_xml.find('head')
         ntb_id = head.find('meta[@name="NTBID"]')
-        self.assertEqual(ntb_id.get('content'), 'NTB' + article['guid'])
+        self.assertEqual(ntb_id.get('content'), 'NTB' + family_id)
         doc_id = nitf_xml.find('head/docdata/doc-id')
         self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(article['guid'], 3))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(family_id, 3))
 
     @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
     def test_description_text_none(self):
@@ -619,86 +620,10 @@ def test_rewrite_sequence_none(self):
         doc = formatter_output[0]['encoded_item']
         nitf_xml = etree.fromstring(doc)
         doc_id = nitf_xml.find('head/docdata/doc-id')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(article['guid'], 0))
+        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(article['family_id'], 0))
 
     @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
     def test_language_empty(self):
         article = copy.deepcopy(self.article)
         article.pop('language')
         self.formatter.format(article, {'name': 'Test NTBNITF'})
-
-
-class NTBNITFFormatterUpdateTest(TestCase):
-    archive = [
-        {
-            '_id': 'urn:localhost.abc',
-            'guid': 'urn:localhost.guid.abc',
-            'family_id': ITEM_ID,
-            'headline': 'test headline',
-            'abstract': TEST_ABSTRACT,
-            'body_html': TEST_BODY,
-            'type': 'text',
-            'priority': '2',
-            "slugline": "this is the slugline œ:?–",
-            'urgency': 2,
-            'versioncreated': NOW,
-            '_current_version': 5,
-            'version': 5,
-            'language': 'nb-NO'
-        },
-        {
-            '_id': 'urn:localhost.abc.up',
-            'guid': 'urn:localhost.guid.abc.up',
-            'family_id': ITEM_ID,
-            'rewrite_sequence': 1,
-            'rewrite_of': 'urn:localhost.abc',
-            'headline': 'test headline',
-            'abstract': TEST_ABSTRACT,
-            'body_html': TEST_BODY,
-            'type': 'text',
-            'priority': '2',
-            "slugline": "this is the slugline œ:?–",
-            'urgency': 2,
-            'versioncreated': NOW,
-            '_current_version': 5,
-            'version': 5,
-            'language': 'nb-NO'
-        },
-        {
-            '_id': 'urn:localhost.abc.up2',
-            'guid': 'urn:localhost.guid.abc.up2',
-            'family_id': ITEM_ID,
-            'rewrite_sequence': 2,
-            'rewrite_of': 'urn:localhost.abc.up',
-            'headline': 'test headline',
-            'abstract': TEST_ABSTRACT,
-            'body_html': TEST_BODY,
-            'type': 'text',
-            'priority': '2',
-            "slugline": "this is the slugline œ:?–",
-            'urgency': 2,
-            'versioncreated': NOW,
-            '_current_version': 5,
-            'version': 5,
-            'language': 'nb-NO'
-        }
-    ]
-
-    @mock.patch.object(SubscribersService, 'generate_sequence_number', lambda self, subscriber: 1)
-    def setUp(self):
-        init_app(self.app)
-        self.app.data.insert('archive', self.archive)
-        self.formatter = NTBNITFFormatter()
-        self.formatter_output = self.formatter.format(self.archive[-1], {'name': 'Test NTBNITF'})
-        self.doc = self.formatter_output[0]['encoded_item']
-        self.nitf_xml = etree.fromstring(self.doc)
-
-    def test_ntbid(self):
-        head = self.nitf_xml.find('head')
-        ntb_id = head.find('meta[@name="NTBID"]')
-        self.assertEqual(ntb_id.get('content'), 'NTB' + self.archive[0]['guid'])
-
-    def test_docid(self):
-        doc_id = self.nitf_xml.find('head/docdata/doc-id')
-        self.assertEqual(doc_id.get('regsrc'), 'NTB')
-        self.assertEqual(doc_id.get('id-string'), 'NTB{}_{:02}'.format(self.archive[0]['guid'], 2))
diff --git a/server/setup.cfg b/server/setup.cfg
@@ -1,6 +1,5 @@
 [flake8]
 max-line-length=120
 exclude=env,bin,lib,include,src
-ignore=F811,D200,D202,D205,D400,D401,D100,D101,D102,D103,D104,D105,D107,W503,W504,W605,F401,E261,F841,
-       B010,B009,B007,B305,B011
+ignore=F811,D200,D202,D205,D400,D401,D100,D101,D102,D103,D104,D105,D107,W503,W504,W605,F401,E261,F841
 # W504, W605, F401, E261 and F841 are temporally ignored, due to recent changes in flake8