Processor resource discovery

CHANGELOG.md

@@ -5,19 +5,38 @@ Versioned according to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
-Fixed:
-
-  * `run_cli`: don't reference undefined vars in error handler, #651
+## [2.22.0b4] - 2021-01-20
 
 Added:
 
+  * Implement file resource algorithm from OCR-D/spec#169, #559
+  * New CLI `ocrd resmgr` to download/browse processor resources, #559
   * `Workspace.rename_file_group` with CLI `ocrd workspace rename-group` to rename file groups, #646
 
 Changed:
 
   * `ocrd workspace add`: guess `--mimetype` if not provided, #658
   * `ocrd workspace add`: warn if `--page-id` not provided, #659
 
+## [2.22.0b3] - 2020-12-30
+
+Fixed:
+  * `name` of resources mustn't contain slash `/`
+
+## [2.22.0b2] - 2020-12-30
+
+Added:
+
+  * PAGE API method `get_AllTextLines`
+  * resources for kraken
+
+## [2.22.0b1] - 2020-12-28
+
+Fixed:
+
+  * `run_cli`: don't reference undefined vars in error handler, #651
+
+
 ## [2.21.0] - 2020-11-27
 
 Changed:
@@ -1249,6 +1268,10 @@ Fixed
 Initial Release
 
 <!-- link-labels -->
+[2.22.0b4]: ../../compare/v2.22.0b4..v2.22.0b3
+[2.22.0b3]: ../../compare/v2.22.0b3..v2.22.0b2
+[2.22.0b2]: ../../compare/v2.22.0b2..v2.22.0b1
+[2.22.0b1]: ../../compare/v2.22.0b1..v2.21.0
 [2.21.0]: ../../compare/v2.21.0..v2.20.2
 [2.20.2]: ../../compare/v2.20.2..v2.20.1
 [2.20.1]: ../../compare/v2.20.1..v2.20.0

Makefile

@@ -64,12 +64,12 @@ deps-ubuntu:
 
 # Install test python deps via pip
 deps-test:
-	$(PIP) install -U "pip>=19.0.0"
+	$(PIP) install -U "pip>=19.0.0,!=20.3.2"
 	$(PIP) install -r requirements_test.txt
 
 # (Re)install the tool
 install:
-	$(PIP) install -U "pip>=19.0.0" wheel
+	$(PIP) install -U "pip>=19.0.0,!=20.3.2" wheel
 	for mod in $(BUILD_ORDER);do (cd $$mod ; $(PIP_INSTALL) .);done
 
 # Install with pip install -e
@@ -148,7 +148,7 @@ assets-server:
 test: assets
 	HOME=$(CURDIR)/ocrd_utils $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging $(TESTDIR)
 	HOME=$(CURDIR) $(PYTHON) -m pytest --continue-on-collection-errors -k TestLogging $(TESTDIR)
-	HOME=$(CURDIR) $(PYTHON) -m pytest --continue-on-collection-errors --ignore=$(TESTDIR)/test_logging.py $(TESTDIR)
+	$(PYTHON) -m pytest --continue-on-collection-errors --ignore=$(TESTDIR)/test_logging.py $(TESTDIR)
 
 test-profile:
 	$(PYTHON) -m cProfile -o profile $$(which pytest)

ocrd/ocrd/cli/__init__.py

@@ -18,6 +18,7 @@ def get_help(self, ctx):
 from ocrd.cli.process import process_cli
 from ocrd.cli.bashlib import bashlib_cli
 from ocrd.cli.validate import validate_cli
+from ocrd.cli.resmgr import resmgr_cli
 from ocrd.decorators import ocrd_loglevel
 from .zip import zip_cli
 from .log import log_cli
@@ -37,3 +38,4 @@ def cli(**kwargs): # pylint: disable=unused-argument
 cli.add_command(zip_cli)
 cli.add_command(validate_cli)
 cli.add_command(log_cli)
+cli.add_command(resmgr_cli)

ocrd/ocrd/cli/resmgr.py

@@ -0,0 +1,124 @@
+import sys
+from os import getcwd
+from os.path import join
+from pathlib import Path
+import requests
+
+import click
+
+from ocrd_utils import (
+    initLogging,
+    getLogger,
+    RESOURCE_LOCATIONS
+)
+from ocrd_validators import OcrdZipValidator
+
+from ..resource_manager import OcrdResourceManager
+
+def print_resources(executable, reslist, resmgr):
+    print('%s' % executable)
+    for resdict in reslist:
+        print('- %s %s (%s)\n  %s' % (
+            resdict['name'],
+            '@ %s' % resmgr.resource_dir_to_location(resdict['path']) if 'path' in resdict else '',
+            resdict['url'],
+            resdict['description']
+        ))
+    print()
+
+@click.group("resmgr")
+def resmgr_cli():
+    """
+    Managing processor resources
+    """
+    initLogging()
+
+@resmgr_cli.command('list-available')
+@click.option('-e', '--executable', help='Show only resources for executable EXEC', metavar='EXEC')
+def list_available(executable=None):
+    """
+    List available resources
+    """
+    resmgr = OcrdResourceManager()
+    for executable, reslist in resmgr.list_available(executable):
+        print_resources(executable, reslist, resmgr)
+
+@resmgr_cli.command('list-installed')
+@click.option('-e', '--executable', help='Show only resources for executable EXEC', metavar='EXEC')
+def list_installed(executable=None):
+    """
+    List installed resources
+    """
+    resmgr = OcrdResourceManager()
+    ret = []
+    for executable, reslist in resmgr.list_installed(executable):
+        print_resources(executable, reslist, resmgr)
+
+@resmgr_cli.command('download')
+@click.option('-n', '--any-url', help='Allow downloading/copying unregistered resources', is_flag=True)
+@click.option('-o', '--overwrite', help='Overwrite existing resources', is_flag=True)
+@click.option('-l', '--location', help='Where to store resources', type=click.Choice(RESOURCE_LOCATIONS), default='data', show_default=True)
+@click.argument('executable', required=True)
+@click.argument('url_or_name', required=True)
+def download(any_url, overwrite, location, executable, url_or_name):
+    """
+    Download resource URL_OR_NAME for processor EXECUTABLE.
+
+    URL_OR_NAME can either be the ``name`` or ``url`` of a registered resource.
+
+    If URL_OR_NAME is '*' (asterisk), download all known resources for this processor
+
+    If ``--any-url`` is given, also accepts URL or filenames of non-registered resources for ``URL_OR_NAME``.
+    """
+    log = getLogger('ocrd.cli.resmgr')
+    resmgr = OcrdResourceManager()
+    basedir = resmgr.location_to_resource_dir(location)
+    is_url = url_or_name.startswith('https://') or url_or_name.startswith('http://')
+    is_filename = Path(url_or_name).exists()
+    find_kwargs = {'executable': executable}
+    if url_or_name != '*':
+        find_kwargs['url' if is_url else 'name'] = url_or_name
+    reslist = resmgr.find_resources(**find_kwargs)
+    if not reslist:
+        log.info("No resources found in registry")
+        if any_url and (is_url or is_filename):
+            log.info("%s unregistered resource %s" % ("Downloading" if is_url else "Copying", url_or_name))
+            if is_url:
+                with requests.get(url_or_name, stream=True) as r:
+                    content_length = int(r.headers.get('content-length'))
+            else:
+                url_or_name = str(Path(url_or_name).resolve())
+                content_length = Path(url_or_name).stat().st_size
+            with click.progressbar(length=content_length, label="Downloading" if is_url else "Copying") as bar:
+                fpath = resmgr.download(
+                    executable,
+                    url_or_name,
+                    overwrite=overwrite,
+                    basedir=basedir,
+                    progress_cb=lambda delta: bar.update(delta))
+            log.info("%s resource '%s' (%s) not a known resource, creating stub in %s'" % (executable, fpath.name, url_or_name, resmgr.user_list))
+            resmgr.add_to_user_database(executable, fpath, url_or_name)
+            log.info("%s %s to %s" % ("Downloaded" if is_url else "Copied", url_or_name, fpath))
+            log.info("Use in parameters as '%s'" % fpath.name)
+        else:
+            sys.exit(1)
+    else:
+        for _, resdict in reslist:
+            if resdict['url'] == '???':
+                log.info("Cannot download user resource %s" % (resdict['name'])),
+                continue
+            log.info("Downloading resource %s" % resdict)
+            with click.progressbar(length=resdict['size']) as bar:
+                fpath = resmgr.download(
+                    executable,
+                    resdict['url'],
+                    name=resdict['name'],
+                    resource_type=resdict['type'],
+                    path_in_archive=resdict.get('path_in_archive', '.'),
+                    overwrite=overwrite,
+                    basedir=basedir,
+                    progress_cb=lambda delta: bar.update(delta)
+                )
+            log.info("Downloaded %s to %s" % (resdict['url'], fpath))
+            log.info("Use in parameters as '%s'" % resmgr.parameter_usage(resdict['name'], usage=resdict['parameter_usage']))
+

ocrd/ocrd/constants.py

@@ -9,12 +9,16 @@
     'DOWNLOAD_DIR',
     'DEFAULT_REPOSITORY_URL',
     'BASHLIB_FILENAME',
+    'RESOURCE_LIST_FILENAME',
     'BACKUP_DIR',
+    'RESOURCE_USER_LIST_COMMENT',
 ]
 
 TMP_PREFIX = 'ocrd-core-'
 DEFAULT_UPLOAD_FOLDER = '/tmp/uploads-ocrd-core'
 DOWNLOAD_DIR = '/tmp/ocrd-core-downloads'
 DEFAULT_REPOSITORY_URL = 'http://localhost:5000/'
 BASHLIB_FILENAME = resource_filename(__name__, 'lib.bash')
+RESOURCE_LIST_FILENAME = resource_filename(__name__, 'resource_list.yml')
+RESOURCE_USER_LIST_COMMENT = "# OCR-D private resource list (consider sending a PR with your own resources to OCR-D/core)"
 BACKUP_DIR = '.backup'

ocrd/ocrd/decorators/__init__.py

@@ -28,13 +28,22 @@ def ocrd_cli_wrap_processor(
     help=False, # pylint: disable=redefined-builtin
     version=False,
     overwrite=False,
+    show_resource=None,
+    list_resources=False,
     **kwargs
 ):
     if not sys.argv[1:]:
         processorClass(workspace=None, show_help=True)
         sys.exit(1)
-    if dump_json or help or version:
-        processorClass(workspace=None, dump_json=dump_json, show_help=help, show_version=version)
+    if dump_json or help or version or show_resource or list_resources:
+        processorClass(
+            workspace=None,
+            dump_json=dump_json,
+            show_help=help,
+            show_version=version,
+            show_resource=show_resource,
+            list_resources=list_resources
+        )
         sys.exit()
     else:
         initLogging()

ocrd/ocrd/decorators/ocrd_cli_options.py

@@ -15,6 +15,7 @@ def ocrd_cli_options(f):
         def cli(mets_url):
             print(mets_url)
     """
+    # XXX Note that the `--help` output is statically generate_processor_help
     params = [
         option('-m', '--mets', help="METS to process", default="mets.xml"),
         option('-w', '--working-dir', help="Working Directory"),
@@ -25,6 +26,8 @@ def cli(mets_url):
         option('-O', '--output-file-grp', help='File group(s) used as output.', default='OUTPUT'),
         option('-g', '--page-id', help="ID(s) of the pages to process"),
         option('--overwrite', help="Overwrite the output file group or a page range (--page-id)", is_flag=True, default=False),
+        option('-C', '--show-resource', help='Dump the content of processor resource RESNAME', metavar='RESNAME'),
+        option('-L', '--list-resources', is_flag=True, default=False, help='List names of processor resources'),
         parameter_option,
         parameter_override_option,
         option('-J', '--dump-json', help="Dump tool description as JSON and exit", is_flag=True, default=False),

ocrd/ocrd/processor/base.py

@@ -2,13 +2,34 @@
 Processor base class and helper functions
 """
 
-__all__ = ['Processor', 'generate_processor_help', 'run_cli', 'run_processor']
+__all__ = [
+    'Processor',
+    'generate_processor_help',
+    'run_cli',
+    'run_processor'
+]
 
-import os
+from os import makedirs
+from os.path import exists, isdir, join
+from shutil import copyfileobj
 import json
-from ocrd_utils import VERSION as OCRD_VERSION, MIMETYPE_PAGE, getLogger
+import os
+import re
+import sys
+
+import requests
+
+from ocrd_utils import (
+    VERSION as OCRD_VERSION,
+    MIMETYPE_PAGE,
+    getLogger,
+    initLogging,
+    list_resource_candidates,
+    list_all_resources,
+)
 from ocrd_validators import ParameterValidator
 from ocrd_models.ocrd_page import MetadataItemType, LabelType, LabelsType
+from ..resource_manager import OcrdResourceManager
 
 # XXX imports must remain for backwards-compatibilty
 from .helpers import run_cli, run_processor, generate_processor_help # pylint: disable=unused-import
@@ -33,6 +54,8 @@ def __init__(
             input_file_grp="INPUT",
             output_file_grp="OUTPUT",
             page_id=None,
+            show_resource=None,
+            list_resources=False,
             show_help=False,
             show_version=False,
             dump_json=False,
@@ -43,6 +66,20 @@ def __init__(
         if dump_json:
             print(json.dumps(ocrd_tool, indent=True))
             return
+        if list_resources:
+            for res in list_all_resources(ocrd_tool['executable']):
+                print(res)
+            return
+        if show_resource:
+            res_fname = list_resource_candidates(ocrd_tool['executable'], show_resource, is_file=True)
+            if not res_fname:
+                initLogging()
+                logger = getLogger('ocrd.%s.__init__' % ocrd_tool['executable'])
+                logger.error("Failed to resolve %s for processort %s" % (show_resource, ocrd_tool['executable']))
+            else:
+                with open(res_fname[0], 'rb') as f:
+                    copyfileobj(f, sys.stdout.buffer)
+            return
         self.ocrd_tool = ocrd_tool
         if show_help:
             self.show_help()
@@ -84,6 +121,7 @@ def process(self):
         """
         raise Exception("Must be implemented")
 
+
     def add_metadata(self, pcgts):
         """
         Adds PAGE-XML MetadataItem describing the processing step
@@ -107,6 +145,41 @@ def add_metadata(self, pcgts):
                                          value=OCRD_VERSION)])
                     ]))
 
+    def resolve_resource(self, val):
+        """
+        Resolve a resource name to an absolute file path with the algorithm in
+        https://ocr-d.de/en/spec/ocrd_tool#file-parameters
+
+        Args:
+            val (string): resource value to resolve
+        """
+        executable = self.ocrd_tool['executable']
+        if exists(val):
+            return val
+        ret = [cand for cand in list_resource_candidates(executable, val) if exists(cand)]
+        if ret:
+            return ret[0]
+        resmgr = OcrdResourceManager()
+        reslist = resmgr.find_resources(executable, name=val)
+        if not reslist:
+            reslist = resmgr.find_resources(executable, url=val)
+        if not reslist:
+            raise FileNotFoundError("Could not resolve %s resource '%s'" % (executable, val))
+        _, resdict = reslist[0]
+        return str(resmgr.download(
+            executable,
+            url=resdict['url'],
+            name=resdict['name'],
+            path_in_archive=resdict['path_in_archive'],
+            resource_type=resdict['type']
+        ))
+
+    def list_all_resources(self):
+        """
+        List all resources found in the filesystem
+        """
+        return list_all_resources(self.ocrd_tool['executable'])
+
     @property
     def input_files(self):
         """