From 10443d9a9811834bcfe6717b29648730cddef57a Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Wed, 2 Nov 2016 00:00:21 +0000 Subject: [PATCH 1/9] [#372] pydmtxlib --- .gitignore | 1 + CHANGELOG.md | 1 + DevelopingOnLinux.md | 17 -------- DevelopingOnMacOSX.md | 34 ++-------------- build.sh | 14 +++---- inselect.spec | 10 +++-- inselect/scripts/read_barcodes.py | 5 ++- read_barcodes.spec | 66 +++++++++++++++++++++++++++++++ requirements.pip | 1 + requirements.win32 | 1 - requirements.win64 | 1 - setup.py | 2 +- 12 files changed, 90 insertions(+), 63 deletions(-) create mode 100644 read_barcodes.spec diff --git a/.gitignore b/.gitignore index 456db90..935f02e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ cover MANIFEST *spec !inselect.spec +!read_barcodes.spec !segment.spec *tar.gz *dmg diff --git a/CHANGELOG.md b/CHANGELOG.md index 6169554..3d2626b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ This is an overview of major changes. Refer to the git repository for a full log Version 0.1.35 ------------- +- #372 Support pydmtxlib - #273 Inselect as a package - #83 Architecture and code organisation diff --git a/DevelopingOnLinux.md b/DevelopingOnLinux.md index 84c49fd..9d74c3b 100644 --- a/DevelopingOnLinux.md +++ b/DevelopingOnLinux.md @@ -35,23 +35,6 @@ mkvirtualenv --system-site-packages inselect pip2 install -r requirements.pip ``` -## LibDMTX barcode reading library - -* Get source for the wrappers - - ``` - cd ~/projects - git clone git://libdmtx.git.sourceforge.net/gitroot/libdmtx/dmtx-wrappers - ``` - -* Install Python library - - ``` - cd dmtx-wrappers/python - python2 setup.py install - cd ~/projects/inselect - ``` - ## Test barcode reading libraries Inselect has optional barcode reading capabilities. The dependent libraries diff --git a/DevelopingOnMacOSX.md b/DevelopingOnMacOSX.md index 53e5d27..e3eab66 100644 --- a/DevelopingOnMacOSX.md +++ b/DevelopingOnMacOSX.md @@ -32,44 +32,16 @@ pip install -r requirements.pip ## LibDMTX barcode reading library -* Get source for the library and the wrappers +* Install the `libdmtx` shared lib ``` - cd ~/projects - git clone git://libdmtx.git.sourceforge.net/gitroot/libdmtx/libdmtx - git clone git://libdmtx.git.sourceforge.net/gitroot/libdmtx/dmtx-wrappers - ``` - -* Build library - - ``` - cd libdmtx - git checkout v0.7.4 - ./autogen.sh - ./configure - make - ``` - -* Build Python library - - ``` - cd ../dmtx-wrappers/ - ./autogen.sh - ./configure - make - ``` - -* Install Python library - - ``` - cd python - python setup.py install + brew install libdmtx ``` * Test ``` - python -c "import pydmtx; print(pydmtx)" + python -c "import libdmtx; print(libdmtx)" ``` ## Test barcode reading libraries diff --git a/build.sh b/build.sh index 19a58fa..446e0d6 100755 --- a/build.sh +++ b/build.sh @@ -22,7 +22,7 @@ python -c "from gouda.engines import LibDMTXEngine; assert LibDMTXEngine.availab echo Report startup time and check for non-essential binary imports mkdir build time python -v inselect.py --quit &> build/startup_log -for module in cv2 numpy pydmtx scipy sklearn zbar; do +for module in cv2 numpy libdmtx scipy sklearn zbar; do if grep -q $module build/startup_log; then echo Non-essential binary $module imported on startup exit 1 @@ -37,17 +37,17 @@ echo Source build mv dist/inselect-$VERSION.tar.gz . if [[ "$OSTYPE" == "darwin"* ]]; then - # Clean existing build files - pyinstaller --clean inselect.spec + # Scripts that have additional requirements in theie own spec files + for script in inselect read_barcodes segment; do + pyinstaller --clean $script.spec + done - for script in export_metadata ingest read_barcodes save_crops; do + # Scripts for which the .spec file can be generated + for script in export_metadata ingest save_crops; do rm -rf $script.spec pyinstaller --onefile --hidden-import numpy inselect/scripts/$script.py done - # segment has additional requirements so have its own spec file - pyinstaller --clean segment.spec - # Add a few items to the PropertyList file generated by PyInstaller python -m bin.plist dist/inselect.app/Contents/Info.plist # Example document diff --git a/inselect.spec b/inselect.spec index cfd8345..e31a3d2 100644 --- a/inselect.spec +++ b/inselect.spec @@ -8,9 +8,9 @@ block_cipher = None a = Analysis(['inselect.py'], pathex=[str(Path('.').absolute())], - binaries=None, + binaries=[], datas=[('inselect/inselect.qss', '')], - hiddenimports=['sklearn.neighbors.typedefs'], + hiddenimports=['sklearn.neighbors.typedefs', 'libdmtx'], hookspath=[], runtime_hooks=[], excludes=['Tkinter'], @@ -19,11 +19,15 @@ a = Analysis(['inselect.py'], cipher=block_cipher) +# libdmtx dylib is not detected because it is loaded by a ctypes call +a.binaries += TOC([ + ('libdmtx.dylib', '/usr/local/Cellar/libdmtx/0.7.4/lib/libdmtx.dylib', 'BINARY'), +]) + # PyInstaller does not detect some dylibs, I think in some cases because they # are symlinked. # See Stack Overflow post http://stackoverflow.com/a/17595149 for example # of manipulating Analysis.binaries. - MISSING_DYLIBS = ( 'libiomp5.dylib', 'libmkl_intel_lp64.dylib', diff --git a/inselect/scripts/read_barcodes.py b/inselect/scripts/read_barcodes.py index 8c6a2c7..b5bd01a 100755 --- a/inselect/scripts/read_barcodes.py +++ b/inselect/scripts/read_barcodes.py @@ -18,11 +18,12 @@ try: import gouda +except ImportError: + gouda = engine_options = resize = roi = None +else: from gouda.engines.options import engine_options from gouda.strategies.resize import resize from gouda.strategies.roi.roi import roi -except ImportError: - gouda = engine_options = resize = roi = None class BarcodeReader(object): diff --git a/read_barcodes.spec b/read_barcodes.spec new file mode 100644 index 0000000..ad76c2c --- /dev/null +++ b/read_barcodes.spec @@ -0,0 +1,66 @@ +import sys + +from pathlib import Path + +block_cipher = None + + +a = Analysis(['inselect/scripts/read_barcodes.py'], + pathex=[str(Path('.').absolute())], + binaries=[], + datas=None, + hiddenimports=['numpy', 'libdmtx'], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher) + + +# libdmtx dylib is not detected because it is loaded by a ctypes call +a.binaries += TOC([ + ('libdmtx.dylib', '/usr/local/Cellar/libdmtx/0.7.4/lib/libdmtx.dylib', 'BINARY'), +]) + +# PyInstaller does not detect some dylibs, I think in some cases because they +# are symlinked. +# See Stack Overflow post http://stackoverflow.com/a/17595149 for example +# of manipulating Analysis.binaries. +MISSING_DYLIBS = ( + 'libopencv_contrib.2.4.dylib', + 'libopencv_nonfree.2.4.dylib', + 'libopencv_gpu.2.4.dylib', + 'libopencv_legacy.2.4.dylib', + 'libopencv_photo.2.4.dylib', + 'libopencv_ocl.2.4.dylib', + 'libopencv_calib3d.2.4.dylib', + 'libopencv_features2d.2.4.dylib', + 'libopencv_flann.2.4.dylib', + 'libopencv_ml.2.4.dylib', + 'libopencv_video.2.4.dylib', + 'libopencv_objdetect.2.4.dylib', + 'libopencv_highgui.2.4.dylib', + 'libopencv_imgproc.2.4.dylib', + 'libopencv_core.2.4.dylib', +) + +# The lib directory associated with this environment +LIB = Path(sys.argv[0]).parent.parent.joinpath('lib') + +# Find the source for each library and add it to the list of binaries +a.binaries += TOC([ + (lib, str(LIB.joinpath(lib).resolve()), 'BINARY') for lib in MISSING_DYLIBS +]) +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + name='read_barcodes', + debug=False, + strip=False, + upx=True, + console=True ) diff --git a/requirements.pip b/requirements.pip index 69aa9bd..f48dd1d 100644 --- a/requirements.pip +++ b/requirements.pip @@ -4,6 +4,7 @@ exifread==2.1.2 humanize==0.5.1 pathlib==1.0.1 psutil==4.0.0 +pylibdmtx==0.1.0 python-dateutil==2.3 # 2.5.0 causes problems for PyInstaller builds on Mac OS X pytz==2015.7 PyYAML==3.11 diff --git a/requirements.win32 b/requirements.win32 index f3b91e6..eb3113e 100644 --- a/requirements.win32 +++ b/requirements.win32 @@ -1,2 +1 @@ -https://github.com/NaturalHistoryMuseum/dmtx-wrappers/releases/download/v0.7.4b1/pydmtx-0.7.4b1-cp27-none-win32.whl https://github.com/NaturalHistoryMuseum/zbar-python-patched/releases/download/v0.10/zbar-0.10-cp27-none-win32.whl \ No newline at end of file diff --git a/requirements.win64 b/requirements.win64 index 5d8d8aa..d53a70b 100644 --- a/requirements.win64 +++ b/requirements.win64 @@ -1,2 +1 @@ -https://github.com/NaturalHistoryMuseum/dmtx-wrappers/releases/download/v0.7.4b1/pydmtx-0.7.4b1-cp27-none-win_amd64.whl https://github.com/NaturalHistoryMuseum/ZBarWin64/releases/download/v0.10/zbar-0.10-cp27-none-win_amd64.whl diff --git a/setup.py b/setup.py index fefae3a..97b7576 100755 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ 'install_requires': REQUIREMENTS, 'extras_require': { 'gui': ['exifread>=2.1.2', 'humanize>=0.5.1', 'psutil>=4.0.0', 'PySide>=1.2.1'], - 'barcodes': ['gouda>=0.1.6', 'pydmtx>=0.7.4b1', 'zbar>=0.10'], + 'barcodes': ['gouda>=0.1.8', 'pylibdmtx>=0.1.0', 'zbar>=0.10'], 'windows': ['pywin32>=220'], 'development': ['coveralls>=0.4.1', 'mock>=1.0.1', 'nose>=1.3.4'], }, From ff4732d659ce966f64f6390f393df200fb114103 Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Thu, 3 Nov 2016 15:24:51 +0000 Subject: [PATCH 2/9] Better comments and a minor correction --- inselect.spec | 7 ++++--- read_barcodes.spec | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/inselect.spec b/inselect.spec index e31a3d2..0c48572 100644 --- a/inselect.spec +++ b/inselect.spec @@ -10,7 +10,7 @@ a = Analysis(['inselect.py'], pathex=[str(Path('.').absolute())], binaries=[], datas=[('inselect/inselect.qss', '')], - hiddenimports=['sklearn.neighbors.typedefs', 'libdmtx'], + hiddenimports=['sklearn.neighbors.typedefs'], hookspath=[], runtime_hooks=[], excludes=['Tkinter'], @@ -19,12 +19,13 @@ a = Analysis(['inselect.py'], cipher=block_cipher) -# libdmtx dylib is not detected because it is loaded by a ctypes call +# libdmtx dylib is not detected because it is loaded by a ctypes call in +# pylibdmtx a.binaries += TOC([ ('libdmtx.dylib', '/usr/local/Cellar/libdmtx/0.7.4/lib/libdmtx.dylib', 'BINARY'), ]) -# PyInstaller does not detect some dylibs, I think in some cases because they +# PyInstaller does not detect some dylibs, in some cases (I think) because they # are symlinked. # See Stack Overflow post http://stackoverflow.com/a/17595149 for example # of manipulating Analysis.binaries. diff --git a/read_barcodes.spec b/read_barcodes.spec index ad76c2c..cce5d9c 100644 --- a/read_barcodes.spec +++ b/read_barcodes.spec @@ -9,7 +9,7 @@ a = Analysis(['inselect/scripts/read_barcodes.py'], pathex=[str(Path('.').absolute())], binaries=[], datas=None, - hiddenimports=['numpy', 'libdmtx'], + hiddenimports=['numpy'], hookspath=[], runtime_hooks=[], excludes=[], @@ -18,12 +18,13 @@ a = Analysis(['inselect/scripts/read_barcodes.py'], cipher=block_cipher) -# libdmtx dylib is not detected because it is loaded by a ctypes call +# libdmtx dylib is not detected because it is loaded by a ctypes call in +# pylibdmtx a.binaries += TOC([ ('libdmtx.dylib', '/usr/local/Cellar/libdmtx/0.7.4/lib/libdmtx.dylib', 'BINARY'), ]) -# PyInstaller does not detect some dylibs, I think in some cases because they +# PyInstaller does not detect some dylibs, in some cases (I think) because they # are symlinked. # See Stack Overflow post http://stackoverflow.com/a/17595149 for example # of manipulating Analysis.binaries. From aacb8c05e6b4807d8b85339b1a20f0e2f27dda7d Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 4 Nov 2016 09:54:49 +0000 Subject: [PATCH 3/9] Freeze correct libdmtx dll --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 97b7576..158a071 100755 --- a/setup.py +++ b/setup.py @@ -83,6 +83,12 @@ ('{site_packages}/numpy', 'numpy'), ('{site_packages}/scipy', 'scipy'), ('{site_packages}/sklearn', 'sklearn'), + ( + '{environment_root}/' + 'libdmtx-{0}.dll'.format( + '64' if sys.maxsize > 2**32 else '32' + ), + 'libdmtx-{0}.dll'.format('64' if sys.maxsize > 2**32 else '32') + ), ('{environment_root}/Library/bin/mkl_core.dll', 'mkl_core.dll'), ('{environment_root}/Library/bin/libiomp5md.dll', 'libiomp5md.dll'), ('{project_root}/inselect/inselect.qss', 'inselect.qss'), From 15f769d0c22e0c74b9280d806b82f981287abf9f Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 4 Nov 2016 10:50:38 +0000 Subject: [PATCH 4/9] 80 columns --- setup.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 158a071..e20c6f6 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ REQUIREMENTS = [ -# 'cv2>=2.4.8,<3', + # TODO How to specify OpenCV? 'cv2>=2.4.8,<3', 'pathlib>=1.0.1,<1.1', 'Pillow>=3.2.0,<3.4', 'python-dateutil>=2.3,<=2.6', @@ -24,7 +24,8 @@ setup_data = { 'name': 'inselect', 'version': inselect.__version__, - 'author': u'Lawrence Hudson, Alice Heaton, Pieter Holtzhausen, Stéfan van der Walt', + 'author': (u'Lawrence Hudson, Alice Heaton, Pieter Holtzhausen, ' + u'Stéfan van der Walt'), 'author_email': 'l.hudson@nhm.ac.uk', 'maintainer': 'Lawrence Hudson', 'maintainer_email': 'l.hudson@nhm.ac.uk', @@ -42,7 +43,10 @@ 'scripts': ['inselect/scripts/{0}.py'.format(script) for script in SCRIPTS], 'install_requires': REQUIREMENTS, 'extras_require': { - 'gui': ['exifread>=2.1.2', 'humanize>=0.5.1', 'psutil>=4.0.0', 'PySide>=1.2.1'], + 'gui': [ + 'exifread>=2.1.2', 'humanize>=0.5.1', 'psutil>=4.0.0', + 'PySide>=1.2.1' + ], 'barcodes': ['gouda>=0.1.8', 'pylibdmtx>=0.1.0', 'zbar>=0.10'], 'windows': ['pywin32>=220'], 'development': ['coveralls>=0.4.1', 'mock>=1.0.1', 'nose>=1.3.4'], @@ -121,12 +125,10 @@ def cx_setup(): 'environment_root': Path(sys.executable).parent, 'project_root': Path(__file__).parent, } - include_files = [] - for i in setup_data['win32']['include_files']: - include_files.append(( - i[0].format(**format_strings), - i[1] - )) + include_files = [ + (source.format(**format_strings), destination) + for source, destination in setup_data['win32']['include_files'] + ] # Setup setup( @@ -134,7 +136,10 @@ def cx_setup(): version=setup_data['version'], options={ 'build_exe': { - 'packages': setup_data['packages'] + setup_data['win32']['extra_packages'], + 'packages': ( + setup_data['packages'] + + setup_data['win32']['extra_packages'] + ), 'excludes': setup_data['win32']['excludes'], 'include_files': include_files, 'include_msvcr': True, @@ -144,7 +149,9 @@ def cx_setup(): 'upgrade_code': '{fe2ed61d-cd5e-45bb-9d16-146f725e522f}' } }, - executables=[Executable(**i) for i in setup_data['win32']['executables']] + executables=[ + Executable(**i) for i in setup_data['win32']['executables'] + ] ) From e3aeb00ab77b515bb89cc39bd1430a90d7138558 Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 4 Nov 2016 13:21:22 +0000 Subject: [PATCH 5/9] Public release of pylibdmtx --- requirements.pip | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.pip b/requirements.pip index f48dd1d..a0abdf8 100644 --- a/requirements.pip +++ b/requirements.pip @@ -4,7 +4,7 @@ exifread==2.1.2 humanize==0.5.1 pathlib==1.0.1 psutil==4.0.0 -pylibdmtx==0.1.0 +pylibdmtx==0.1.1 python-dateutil==2.3 # 2.5.0 causes problems for PyInstaller builds on Mac OS X pytz==2015.7 PyYAML==3.11 diff --git a/setup.py b/setup.py index e20c6f6..d55b624 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ 'exifread>=2.1.2', 'humanize>=0.5.1', 'psutil>=4.0.0', 'PySide>=1.2.1' ], - 'barcodes': ['gouda>=0.1.8', 'pylibdmtx>=0.1.0', 'zbar>=0.10'], + 'barcodes': ['gouda>=0.1.8', 'pylibdmtx>=0.1.1', 'zbar>=0.10'], 'windows': ['pywin32>=220'], 'development': ['coveralls>=0.4.1', 'mock>=1.0.1', 'nose>=1.3.4'], }, From 6985a33aeeeeac511a096a070f93351be609361d Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 4 Nov 2016 13:27:24 +0000 Subject: [PATCH 6/9] Correct dependency for libdmtx runtime --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1e84f2e..83bac98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ virtualenv: before_install: - sudo apt-get update - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then - sudo apt-get install -y --fix-missing libzbar-dev pyside-tools; + sudo apt-get install -y --fix-missing libdmtx0a pyside-tools; fi # We do this conditionally because it saves us some downloading if the From ce5dab2b9e2f208d2a24c8e89ea97f48ffb6aac9 Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 4 Nov 2016 13:36:06 +0000 Subject: [PATCH 7/9] Reinstate zbar dependency. Need coffee --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 83bac98..b85871f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ virtualenv: before_install: - sudo apt-get update - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then - sudo apt-get install -y --fix-missing libdmtx0a pyside-tools; + sudo apt-get install -y --fix-missing libzbar-dev libdmtx0a pyside-tools; fi # We do this conditionally because it saves us some downloading if the From 8fb6745f326720a66d63763e3d6a55e43ef3654f Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 4 Nov 2016 14:00:14 +0000 Subject: [PATCH 8/9] gouda 0.1.8 --- requirements.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.pip b/requirements.pip index a0abdf8..bb9899c 100644 --- a/requirements.pip +++ b/requirements.pip @@ -25,7 +25,7 @@ requests==2.11.0 six==1.10.0 # Not from PyPI -https://github.com/NaturalHistoryMuseum/gouda/archive/v0.1.6.tar.gz +https://github.com/NaturalHistoryMuseum/gouda/archive/v0.1.8.tar.gz https://github.com/NaturalHistoryMuseum/zbar-python-patched/archive/v0.10.tar.gz ; sys_platform != 'win32' # Development From 2273aa0c49046273efef5458ba7b1be3052679f4 Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 4 Nov 2016 14:46:53 +0000 Subject: [PATCH 9/9] Correct typo in docstring --- inselect/tests/gui/test_read_barcodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inselect/tests/gui/test_read_barcodes.py b/inselect/tests/gui/test_read_barcodes.py index dc0733f..8697636 100644 --- a/inselect/tests/gui/test_read_barcodes.py +++ b/inselect/tests/gui/test_read_barcodes.py @@ -56,7 +56,7 @@ def test_read_barcodes(self, current_settings, mock_warning): "LibDMTXEngine not available") @patch.object(QMessageBox, 'warning', return_value=QMessageBox.Ok) def test_read_barcodes_no_scanned_image(self, mock_warning): - """The user is informed that barcodes cxannot be read without the + """The user is informed that barcodes cannot be read without the scanned image """ with temp_directory_with_files(TESTDATA / 'barcodes.inselect') as tempdir: