tt3 to ttd conversion (#41)

MSPA-728 ebu-tt-3 to ebu-tt-d conversion

.gitignore

@@ -51,6 +51,9 @@ coverage.xml
 *,cover
 .hypothesis/
 
+# Pytest cache
+.pytest_cache/
+
 # Translations
 *.mo
 *.pot

.travis.yml

@@ -1,44 +1,20 @@
 language: python
 
 python:
-  - '2.7'
+  - '3.7'
 
 branches:
   except:
   - gh-pages
 
-before_install:
-  - echo "Checking for java"
-  - java -version
-  - openssl aes-256-cbc -K $encrypted_f18bb5b3c3d4_key -iv $encrypted_f18bb5b3c3d4_iv -in publish-key.enc -out ~/.ssh/publish-key -d
-  - chmod u=rw,og= ~/.ssh/publish-key
-  - echo "Host github.com" >> ~/.ssh/config
-  - echo "    IdentityFile ~/.ssh/publish-key" >> ~/.ssh/config
-  - sudo apt-get -qq update
-  - sudo apt-get install graphviz
-  - git --version
-  - git remote set-url origin git@github.com:ebu/ebu-tt-live-toolkit.git
-  - git fetch origin -f gh-pages:gh-pages
-
 install:
-  - pip install pip==9.0.1
-  - make
-  - python setup.py develop
-  - pip install coveralls
-  - pip install ghp-import
+  - pip install -U pip
+  - make init
 
 script:
-  - pyxbgen --binding-root=./ebu_tt_live/bindings -m __init__ --schema-root=./ebu_tt_live/xsd/ -r -u ebutt_all.xsd
-  - python setup.py test
-  - export PR=https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/$TRAVIS_PULL_REQUEST
-  - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo `curl -s $PR | jq -r .head.ref`; fi)
-  - echo "TRAVIS_BRANCH=$TRAVIS_BRANCH, PR=$PR, BRANCH=$BRANCH"
-
-after_success:
-  - coveralls
+  - make test
+  - echo "Compiling documentation"
+  - make
+  - sudo apt-get -qq update
+  - sudo apt-get install graphviz
   - python setup.py build_sphinx
-  - if [ $BRANCH == "master" ]; then ghp-import -n -p -m "Update gh-pages." docs/build; fi
-
-notifications:
-  slack:
-    secure: RLs+ufYg4HcZfaTl73tTKKVja8DJ9LGk1WO+7B1qa4Fgpv0c/3uXcM+6tvDXllTlqcfLHgum0vbzvE4yapo0em0g/m75t1pEfkh5Pfrcas8IKAM9jg/xycWdYoAkKLRvNOcS4amO29FlzOHFaSSjX+V8JK8Mpm+QM2eGnqIbJokaXgBf/bQxHqGSRvYCeSisEG0f79Mn/0v5OOlbvmetFfvcQVNZDtgpA1CDbvK99F/vQT3qZ04Rh/vRnlcZyH9PEp/TEEJ0yX0NhHYEDnuBKPc86Ack/YCQpT12Ej2xTWjgoUrqv1pcr+h0ltYPXnnCyETrlbeLgNIMUsGW579MThqXekBZ7byRK2DMfjyL5+UgPSmTOIzEg7Qbc1u0r1bUuEi7nsJ5Vf65QeIAcewVASOUKmda9ag0MidTR/VoAsMtf5MzbDifSa1aBHRxZJToshNJVY9V8/lJLdwsbqsG2H+8v8ZuzyRLNiaFhNQTZQWpRXMipvbTDnnVxwpeFiLAqAwEKO9KIyGijqv2yeVvi4WtYvwzM7/1y/ycu7kTDFemKKuKmW2rzzBKJ30vo79ufMQCmn0riKkpaNxBf8R3EUEr86A9yuFpo6/Qc47i3PkeD7MszDqhEAfxl1+sQssWXOdVy6ALFb7RZ6EczpPvAjYLLO0aPa8OPaZjy3aJr+8=

Makefile

@@ -4,7 +4,7 @@ all: init ui
 
 init:
 	pip install --upgrade -r requirements.txt
-	pyxbgen --binding-root=./ebu_tt_live/bindings -m __init__ --schema-root=./ebu_tt_live/xsd/ -r -u ebutt_all.xsd
+	pyxbgen --binding-root=. -m __init__ --schema-root=./ebu_tt_live/xsd/ -r -u ebutt_all.xsd --module-prefix=ebu_tt_live.bindings
 
 initnpm:
 ifeq ("$(wildcard node_modules)","")
@@ -20,7 +20,7 @@ docs:
 	python setup.py build_sphinx
 
 bindings:
-	pyxbgen --binding-root=./ebu_tt_live/bindings -m __init__ --schema-root=./ebu_tt_live/xsd/ -r -u ebutt_all.xsd
+	pyxbgen --binding-root=. -m __init__ --schema-root=./ebu_tt_live/xsd/ -r -u ebutt_all.xsd --module-prefix=ebu_tt_live.bindings
 
 ui: uiclean uibuild
 

README.md

@@ -18,7 +18,7 @@ If you would like to contribute or join the Slack team, please contact <subtitli
 Preparing the build environment
 ===============================
 
-Make sure you have python 2.7+. Make sure you have python virtual environment capability.
+Make sure you have Python 3+. Make sure you have python virtual environment capability.
 
 If not you can install virtualenv systemwide from your operating system's package repository
 or by pip:
@@ -61,7 +61,7 @@ Windows users
 =============
 
 Windows is not the best friend of Makefiles. So there is a make.bat file for those who would like to develop using
-Windows. Assuming python 2.7 and virtualenv is installed and are on the PATH. To build the project you will also need node.js. Please read the instructions for your system [here](https://nodejs.org/en/download/package-manager/). Then run :
+Windows. Assuming Python 3+ and virtualenv is installed and are on the PATH. To build the project you will also need node.js. Please read the instructions for your system [here](https://nodejs.org/en/download/package-manager/). Then run :
 
     make
 

docs/source/conf.py

@@ -68,18 +68,18 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'EBU-TT-Live Toolkit'
-copyright = u'2017, European Broadcasting Union'
-author = u'European Broadcasting Union'
+project = 'EBU-TT-Live Toolkit'
+copyright = '2017, European Broadcasting Union'
+author = 'European Broadcasting Union'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = u'2.1.1'
+version = '2.1.1'
 # The full version, including alpha/beta/rc tags.
-release = u'2.1.1'
+release = '2.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -279,8 +279,8 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'EBU-TT-LiveToolkit.tex', u'EBU-TT-Live Toolkit Documentation',
-     u'European Broadcasting Union', 'manual'),
+    (master_doc, 'EBU-TT-LiveToolkit.tex', 'EBU-TT-Live Toolkit Documentation',
+     'European Broadcasting Union', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -315,7 +315,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'ebu-tt-livetoolkit', u'EBU-TT-Live Toolkit Documentation',
+    (master_doc, 'ebu-tt-livetoolkit', 'EBU-TT-Live Toolkit Documentation',
      [author], 1)
 ]
 
@@ -330,7 +330,7 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'EBU-TT-LiveToolkit', u'EBU-TT-Live Toolkit Documentation',
+    (master_doc, 'EBU-TT-LiveToolkit', 'EBU-TT-Live Toolkit Documentation',
      author, 'EBU-TT-LiveToolkit', 'One line description of project.',
      'Miscellaneous'),
 ]

docs/source/configurator.rst

@@ -125,6 +125,9 @@ Node type dependent options for [nodeN] : ::
 
    type="distributor" : No options
 
+   type="denester"
+   └─sequence_identifier : sequence identifier, default "Denester1"
+
 Output carriage type dependent options for "carriage": ::
 
    type="direct"

docs/source/denesting.rst

@@ -0,0 +1,37 @@
+Denesting of EBU-TT-Live documents
+======================================
+
+DenesterNode should be used when a EBU-TT-3 document has a div that contains
+other divs, or a span contains another span, and those nested elements need
+to be flattened, for example before conversion to EBU-TT-D. These elements
+are not able to be nested inside each other in EBU-TT-D documents.
+
+When documents are Denested, any nested elements must be removed from
+their parent elements, while retaining attributes they would have inherited.
+To address this, the DenesterNode node processes the
+document(s) with the
+:py:func:`ebu_tt_live.node.denester.DenesterNode.recurse` and
+:py:func:`ebu_tt_live.node.denester.DenesterNode.recurse_span`
+functions. These will iterate through the file to locate the deepest
+nested element, and create a new copy of it with its content and
+expected inherited attributes. The end result is a file containing
+these newly created divs/spans in place of the nested ones.
+
+Once the new divs and spans are created, divs that are sequential in
+the list of divs and have the same attributes are combined into a single
+div. This is done by the
+:py:func:`ebu_tt_live.node.denester.DenesterNode.combine_divs`
+function to reduce the number of divs in the resulting file.
+
+In the combined divs, every p element should have the same region
+as its parent div. Any p elements that specify a different region
+to their inherited region are removed here: as per TTML semantics,
+such p elements are never presented.
+The
+:py:func:`ebu_tt_live.node.denester.DenesterNode.check_p_regions`
+function iterates through the divs that have an assigned region and
+removes any p where its region does not match.
+It then removes the region attribute from any remaining p, as it will
+inhert the region of its parent div and the attribute is unnecessary.
+It will also remove any now-empty divs that exist as a result of having
+their p elements removed.

docs/source/ebu_tt_live.node.rst

@@ -80,3 +80,11 @@ node Package
     :members:
     :undoc-members:
     :show-inheritance:
+
+:mod:`denester` Module
+----------------------
+
+.. automodule:: ebu_tt_live.node.denester
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/inode.puml

@@ -70,6 +70,11 @@ class BufferDelayNode {
   +process_document()
 }
 
+class DenesterNode{
+  ..methods..
+  +process_document()
+}
+
 class RetimingDelayNode{
   ..methods..
   +process_document()

docs/source/overview.rst

@@ -18,4 +18,5 @@ The components mimic the nodes and carriage mechanisms defined in the specificat
     timing_resolution
     segmentation
     deduplication
+    denesting
     conversion

docs/source/scripts_and_their_functions.rst

@@ -107,6 +107,17 @@ attributes are duplicated.
 For the default configuration of the node, see:
 ``ebu-run --admin.conf=ebu_tt_live/examples/config/deduplicator_fs.json``
 
+Denester Node
+-------------
+This node flattens nested ``div`` and ``span`` elements such that no
+``div`` ends up containing a ``div`` and no ``span`` ends up containing
+a ``span``. It also removes any ``p`` elements that specify a ``region``
+attribute that differs from a specified region on an ancester element.
+
+If nested ``div`` or ``span`` elements might be present in a document, the
+Denester node should be used to flatten them before passing them to the
+EBU-TT-D Encoder, because EBU-TT-D does not permit such nested elements.
+
 Retiming Delay Node
 -------------------
 This script modifies the times within each Document and issues them without

ebu_tt_live/adapters/base.py

@@ -9,11 +9,10 @@
 # ==========
 
 
-class IDocumentDataAdapter(object):
+class IDocumentDataAdapter(object, metaclass=AutoRegisteringABCMeta):
     """
     This adapter is used to do various conversions on the payload between the carriage and the node
     """
-    __metaclass__ = AutoRegisteringABCMeta
 
     __impl_registry = {}
     _expects = AbstractStaticMember(validate_types_only)
@@ -24,7 +23,7 @@ def auto_register_impl(cls, impl_class):
         impl_expects = impl_class.expects()
         provides_map = cls.__impl_registry.setdefault(impl_expects, weakref.WeakValueDictionary())
         impl_provides = impl_class.provides()
-        if impl_provides in provides_map.keys():
+        if impl_provides in list(provides_map.keys()):
             log.warning(
                 '({} -> {}) adapter already registered: {}. Ignoring: {} '.format(
                     impl_expects,
@@ -83,14 +82,13 @@ def convert_data(self, data, **kwargs):
         raise NotImplementedError()
 
 
-class INodeCarriageAdapter(object):
+class INodeCarriageAdapter(object, metaclass=AutoRegisteringABCMeta):
     """
     This adapter wraps the DocumentDataAdapter conversion logic and shows a dual interface. It responsibility is
     to facilitate direct communication between incompatible carriage mechanisms and processing nodes.
     This is a tricky business because this class does not have a hardcoded expects-provides interface contract.
     It works it out as it goes forward from the parameters.
     """
-    __metaclass__ = AutoRegisteringABCMeta
 
     @abstractproperty
     def data_adapters(self):

ebu_tt_live/adapters/document_data.py

@@ -106,7 +106,7 @@ def __init__(self, media_clock=None):
         self._converter = EBUTT3EBUTTDConverter(media_clock=media_clock)
 
     def convert_data(self, data, **kwargs):
-        return EBUTTDDocument.create_from_raw_binding(self._converter.convert_document(data)), kwargs
+        return EBUTTDDocument.create_from_raw_binding(self._converter.convert_document(data.binding)), kwargs
 
 
 def get_document_data_adapter(expects, provides):

ebu_tt_live/adapters/test/test_document_data_adapters.py

@@ -91,7 +91,7 @@ def _assert_output_type(self, result):
         self.assertIsInstance(result, self._output_type)
 
     def _assert_kwargs_passtrough(self, result_kwargs, expected_keys):
-        self.assertEquals(set(result_kwargs.keys()), set(expected_keys))
+        self.assertEqual(set(result_kwargs.keys()), set(expected_keys))
 
     def _get_xml(self):
         with open(self._test_xml_path, 'r') as xml_file: