From 8adba457eabde620836d5b485f08a415b9f0f3bb Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Wed, 23 Oct 2013 14:47:32 -0700 Subject: [PATCH 1/3] python 3 compatibility --- bin/catkin_make | 4 ++-- bin/catkin_prepare_release | 4 ++-- cmake/empy.cmake | 2 +- cmake/test/download_checkmd5.py | 7 +++++-- python/catkin/builder.py | 12 ++++++++---- python/catkin/find_in_workspaces.py | 2 +- python/catkin/package_version.py | 2 +- python/catkin/tidy_xml.py | 20 ++++++++++++++------ test/unit_tests/test_builder.py | 5 ++++- test/unit_tests/test_test_results.py | 5 ++++- 10 files changed, 42 insertions(+), 21 deletions(-) diff --git a/bin/catkin_make b/bin/catkin_make index 1d08d60c1..c1211d6bd 100755 --- a/bin/catkin_make +++ b/bin/catkin_make @@ -112,12 +112,12 @@ def main(): whitelist_pkg_names = get_package_names_with_recursive_dependencies(packages, args.only_pkg_with_deps) print('Whitelisted packages: %s' % ', '.join(sorted(whitelist_pkg_names))) - packages = {path: p for path, p in packages.iteritems() if p.name in whitelist_pkg_names} + packages = {path: p for path, p in packages.items() if p.name in whitelist_pkg_names} cmake_args += ['-DCATKIN_WHITELIST_PACKAGES=%s' % ';'.join(sorted(whitelist_pkg_names))] # verify that specified package exists in workspace if args.pkg: - packages_by_name = {p.name: path for path, p in packages.iteritems()} + packages_by_name = {p.name: path for path, p in packages.items()} unknown_packages = [name for name in args.pkg if name not in packages_by_name] if len(unknown_packages) == len(args.pkg): # all package names are unknown diff --git a/bin/catkin_prepare_release b/bin/catkin_prepare_release index 7093b2a72..41ea733d0 100755 --- a/bin/catkin_prepare_release +++ b/bin/catkin_prepare_release @@ -238,7 +238,7 @@ def main(): raise RuntimeError(fmt("@{rf}Aborted release, fix the names of the packages.")) local_modifications = [] - for pkg_path, package in packages.iteritems(): + for pkg_path, package in packages.items(): # verify that the package.xml files don't have modifications pending package_xml_path = os.path.join(pkg_path, PACKAGE_MANIFEST_FILENAME) if has_changes(base_path, package_xml_path, vcs_type): @@ -261,7 +261,7 @@ def main(): # check for changelog entries missing_changelogs = [] missing_changelogs_but_forthcoming = {} - for pkg_path, package in packages.iteritems(): + for pkg_path, package in packages.items(): changelog_path = os.path.join(pkg_path, CHANGELOG_FILENAME) if not os.path.exists(changelog_path): missing_changelogs.append(package.name) diff --git a/cmake/empy.cmake b/cmake/empy.cmake index eee2a4439..06d84ab8c 100644 --- a/cmake/empy.cmake +++ b/cmake/empy.cmake @@ -8,7 +8,7 @@ function(find_python_module module) # A module's location is usually a directory, but for # binary modules # it's a .so file. - execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import re, ${module}; print re.compile('/__init__.py.*').sub('',${module}.__file__)" + execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import re, ${module}; print(re.compile('/__init__.py.*').sub('',${module}.__file__))" RESULT_VARIABLE _${module}_status OUTPUT_VARIABLE _${module}_location ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cmake/test/download_checkmd5.py b/cmake/test/download_checkmd5.py index a183544e4..6b6f0ba58 100755 --- a/cmake/test/download_checkmd5.py +++ b/cmake/test/download_checkmd5.py @@ -3,7 +3,10 @@ from __future__ import print_function import os import sys -import urllib +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve import hashlib from optparse import OptionParser @@ -21,7 +24,7 @@ def download_md5(uri, dest): sys.stdout.write('Downloading %s to %s...' % (uri, dest)) sys.stdout.flush() - urllib.urlretrieve(uri, dest) + urlretrieve(uri, dest) sys.stdout.write('Done\n') diff --git a/python/catkin/builder.py b/python/catkin/builder.py index a18c175a0..ec49a67c9 100644 --- a/python/catkin/builder.py +++ b/python/catkin/builder.py @@ -186,7 +186,11 @@ def run_command(cmd, cwd, quiet=False, colorize=False, add_env=None): out = io.StringIO() if quiet else sys.stdout if capture: while True: - line = unicode(proc.stdout.readline().decode('utf8', 'replace')) + line = proc.stdout.readline().decode('utf8', 'replace') + try: + line = unicode(line) + except NameError: + pass if proc.returncode is not None or not line: break try: @@ -749,11 +753,11 @@ def build_workspace_isolated( whitelist_pkg_names = get_package_names_with_recursive_dependencies(packages, only_pkg_with_deps) print('Whitelisted packages: %s' % ', '.join(sorted(whitelist_pkg_names))) - packages = {path: p for path, p in packages.iteritems() if p.name in whitelist_pkg_names} + packages = {path: p for path, p in packages.items() if p.name in whitelist_pkg_names} # verify that specified package exists in workspace if build_packages: - packages_by_name = {p.name: path for path, p in packages.iteritems()} + packages_by_name = {p.name: path for path, p in packages.items()} unknown_packages = [p for p in build_packages if p not in packages_by_name] if unknown_packages: sys.exit('Packages not found in the workspace: %s' % ', '.join(unknown_packages)) @@ -925,7 +929,7 @@ def cmake_input_changed(packages, build_path, cmake_args=None, filename='catkin_ def get_package_names_with_recursive_dependencies(packages, pkg_names): dependencies = set([]) check_pkg_names = set(pkg_names) - packages_by_name = {p.name: p for path, p in packages.iteritems()} + packages_by_name = {p.name: p for path, p in packages.items()} while check_pkg_names: pkg_name = check_pkg_names.pop() if pkg_name in packages_by_name: diff --git a/python/catkin/find_in_workspaces.py b/python/catkin/find_in_workspaces.py index 769b07569..3ad58d177 100644 --- a/python/catkin/find_in_workspaces.py +++ b/python/catkin/find_in_workspaces.py @@ -129,7 +129,7 @@ def find_in_workspaces(search_dirs=None, project=None, path=None, _workspaces=ge source_paths = get_source_paths(workspace) for source_path in source_paths: packages = find_packages(source_path) - matching_packages = [p for p, pkg in packages.iteritems() if pkg.name == project] + matching_packages = [p for p, pkg in packages.items() if pkg.name == project] if matching_packages: p = os.path.join(source_path, matching_packages[0]) if path is not None: diff --git a/python/catkin/package_version.py b/python/catkin/package_version.py index e6aa6b86f..f89da6805 100644 --- a/python/catkin/package_version.py +++ b/python/catkin/package_version.py @@ -93,7 +93,7 @@ def update_versions(paths, new_version): raise RuntimeError('Could not bump version number in file %s: %s' % (package_path, str(rue))) files[package_path] = new_package_str # if all replacements successful, write back modified package.xml - for package_path, new_package_str in files.iteritems(): + for package_path, new_package_str in files.items(): with open(package_path, 'w') as f: f.write(new_package_str) diff --git a/python/catkin/tidy_xml.py b/python/catkin/tidy_xml.py index c77092d43..c498d94e0 100644 --- a/python/catkin/tidy_xml.py +++ b/python/catkin/tidy_xml.py @@ -40,12 +40,20 @@ # code copied from rosunit/src/junitxml.py -RE_XML_ILLEGAL = '([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ - '|' + \ - '([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ - (unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff), - unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff), - unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff)) +try: + RE_XML_ILLEGAL = '([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ + '|' + \ + '([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ + (unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff), + unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff), + unichr(0xd800), unichr(0xdbff), unichr(0xdc00), unichr(0xdfff)) +except NameError: + RE_XML_ILLEGAL = '([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \ + '|' + \ + '([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \ + (chr(0xd800), chr(0xdbff), chr(0xdc00), chr(0xdfff), + chr(0xd800), chr(0xdbff), chr(0xdc00), chr(0xdfff), + chr(0xd800), chr(0xdbff), chr(0xdc00), chr(0xdfff)) _SAFE_XML_REGEX = re.compile(RE_XML_ILLEGAL) diff --git a/test/unit_tests/test_builder.py b/test/unit_tests/test_builder.py index 4a8cbf80b..41f459956 100644 --- a/test/unit_tests/test_builder.py +++ b/test/unit_tests/test_builder.py @@ -23,7 +23,10 @@ def __init__(self, popen): def readline(self): self.__popen.returncode = 0 - return unichr(2018) + try: + return unichr(2018) + except NameError: + return chr(2018) class MockPopen(object): def __init__(self, *args, **kwargs): diff --git a/test/unit_tests/test_test_results.py b/test/unit_tests/test_test_results.py index dc0735ce5..861f710be 100644 --- a/test/unit_tests/test_test_results.py +++ b/test/unit_tests/test_test_results.py @@ -3,7 +3,10 @@ import unittest import tempfile import shutil -from cStringIO import StringIO +try: + from io import StringIO +except ImportError: + from cStringIO import StringIO try: From 8210efe52c3020eb89815eb7af59e4af9fd66950 Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Wed, 20 Nov 2013 15:04:22 -0800 Subject: [PATCH 2/3] modify Python 3 install location when using Debian layout --- cmake/python.cmake | 2 ++ cmake/templates/python_distutils_install.sh.in | 1 - python/catkin/builder.py | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/python.cmake b/cmake/python.cmake index e83d97db6..6f649a366 100644 --- a/cmake/python.cmake +++ b/cmake/python.cmake @@ -17,6 +17,8 @@ if(SETUPTOOLS_DEB_LAYOUT) message(STATUS "Using Debian Python package layout") set(PYTHON_PACKAGES_DIR dist-packages) set(SETUPTOOLS_ARG_EXTRA "--install-layout=deb") + # use major version only when installing with debian layout + string(SUBSTRING "${PYTHON_VERSION_XDOTY}" 0 1 PYTHON_VERSION_XDOTY) else() message(STATUS "Using default Python package layout") set(PYTHON_PACKAGES_DIR site-packages) diff --git a/cmake/templates/python_distutils_install.sh.in b/cmake/templates/python_distutils_install.sh.in index edc3e33c4..e7b56c72d 100755 --- a/cmake/templates/python_distutils_install.sh.in +++ b/cmake/templates/python_distutils_install.sh.in @@ -14,7 +14,6 @@ fi cd "@INSTALL_CMD_WORKING_DIRECTORY@" -# todo --install-layout=deb per platform # Note that PYTHONPATH is pulled from the environment to support installing # into one location when some dependencies were installed in another # location, #123. diff --git a/python/catkin/builder.py b/python/catkin/builder.py index ec49a67c9..0e16cbc03 100644 --- a/python/catkin/builder.py +++ b/python/catkin/builder.py @@ -233,11 +233,13 @@ def isolation_print_command(cmd, path=None, add_env=None): def get_python_install_dir(): # this function returns the same value as the CMake variable PYTHON_INSTALL_DIR from catkin/cmake/python.cmake python_install_dir = 'lib' + python_use_debian_layout = os.path.exists('/etc/debian_version') if os.name != 'nt': - python_version_xdoty = str(sys.version_info[0]) + '.' + str(sys.version_info[1]) + python_version_xdoty = str(sys.version_info[0]) + if not python_use_debian_layout: + python_version_xdoty += '.' + str(sys.version_info[1]) python_install_dir = os.path.join(python_install_dir, 'python' + python_version_xdoty) - python_use_debian_layout = os.path.exists('/etc/debian_version') python_packages_dir = 'dist-packages' if python_use_debian_layout else 'site-packages' python_install_dir = os.path.join(python_install_dir, python_packages_dir) return python_install_dir From 0436f6c932d1a1142001e854edb24bd2e3fe8d3a Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Wed, 20 Nov 2013 15:07:30 -0800 Subject: [PATCH 3/3] add comments about python 2 compatibility --- python/catkin/builder.py | 1 + test/unit_tests/test_builder.py | 1 + 2 files changed, 2 insertions(+) diff --git a/python/catkin/builder.py b/python/catkin/builder.py index 0e16cbc03..dad67ac47 100644 --- a/python/catkin/builder.py +++ b/python/catkin/builder.py @@ -188,6 +188,7 @@ def run_command(cmd, cwd, quiet=False, colorize=False, add_env=None): while True: line = proc.stdout.readline().decode('utf8', 'replace') try: + # for Python 2 compatibility only line = unicode(line) except NameError: pass diff --git a/test/unit_tests/test_builder.py b/test/unit_tests/test_builder.py index 41f459956..539868bad 100644 --- a/test/unit_tests/test_builder.py +++ b/test/unit_tests/test_builder.py @@ -24,6 +24,7 @@ def __init__(self, popen): def readline(self): self.__popen.returncode = 0 try: + # for Python 2 compatibility only return unichr(2018) except NameError: return chr(2018)