diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index ab169c06764..76cf6b715fe 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -1,11 +1,13 @@ ansicolors==1.0.2 beautifulsoup4>=4.3.2,<4.4 cffi==1.11.1 +configparser==3.5.0 contextlib2==0.5.5 coverage>=4.5,<4.6 docutils>=0.12,<0.13 fasteners==0.14.1 faulthandler==2.6 +future==0.16.0 futures==3.0.5 isort==4.2.5 Markdown==2.1.1 diff --git a/pants-plugins/src/python/internal_backend/sitegen/tasks/sitegen.py b/pants-plugins/src/python/internal_backend/sitegen/tasks/sitegen.py index 1c2cc24b8fa..45af7aa4c53 100644 --- a/pants-plugins/src/python/internal_backend/sitegen/tasks/sitegen.py +++ b/pants-plugins/src/python/internal_backend/sitegen/tasks/sitegen.py @@ -5,6 +5,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import cgi import collections import json import os @@ -354,7 +355,9 @@ def generate_generated(config, here): def render_html(dst, config, soups, precomputed, template): soup = soups[dst] - renderer = Renderer() + renderer = Renderer(escape=cgi.escape) # TODO(python3port): cgi.escape is deprecated in favor html.escape + # (the default used by Pystache on Python3). html.escape() escapes single + # quotes, whereas cgi.escape() does not. This will need to be addressed. title = precomputed.page[dst].title topdots = ('../' * dst.count('/')) if soup.body: diff --git a/src/python/pants/base/BUILD b/src/python/pants/base/BUILD index 40fcc718f77..39f56445974 100644 --- a/src/python/pants/base/BUILD +++ b/src/python/pants/base/BUILD @@ -5,6 +5,7 @@ python_library( name = 'build_environment', sources = ['build_environment.py'], dependencies = [ + '3rdparty/python:future', ':build_root', 'src/python/pants/scm', 'src/python/pants/scm:git', @@ -75,6 +76,7 @@ python_library( name = 'fingerprint_strategy', sources = ['fingerprint_strategy.py'], dependencies = [ + '3rdparty/python:future', ':deprecated', 'src/python/pants/util:meta', ] @@ -85,6 +87,7 @@ python_library( sources = ['generator.py'], dependencies = [ ':mustache', + '3rdparty/python:future', '3rdparty/python:pystache', ] ) @@ -92,12 +95,16 @@ python_library( python_library( name = 'hash_utils', sources = ['hash_utils.py'], + dependencies = [ + '3rdparty/python:future', + ] ) python_library( name = 'mustache', sources = ['mustache.py'], dependencies = [ + '3rdparty/python:future', '3rdparty/python:pystache', '3rdparty/python:six', ] @@ -106,17 +113,24 @@ python_library( python_library( name = 'parse_context', sources = ['parse_context.py'], + dependencies = [ + '3rdparty/python:future', + ] ) python_library( name = 'payload', sources = ['payload.py'], + dependencies = [ + '3rdparty/python:future', + ] ) python_library( name = 'payload_field', sources = ['payload_field.py'], dependencies = [ + '3rdparty/python:future', '3rdparty/python/twitter/commons:twitter.common.collections', ':deprecated', ':hash_utils', @@ -136,12 +150,16 @@ python_library( python_library( name = 'revision', sources = ['revision.py'], + dependencies = [ + '3rdparty/python:future', + ] ) python_library( name = 'run_info', sources = ['run_info.py'], dependencies = [ + '3rdparty/python:future', ':build_environment', 'src/python/pants/util:dirutil', 'src/python/pants:version', @@ -152,6 +170,7 @@ python_library( name = 'cmd_line_spec_parser', sources = ['cmd_line_spec_parser.py'], dependencies = [ + '3rdparty/python:future', '3rdparty/python/twitter/commons:twitter.common.collections', ':build_file', ':specs', @@ -171,6 +190,7 @@ python_library( name = 'worker_pool', sources = ['worker_pool.py'], dependencies = [ + '3rdparty/python:future', 'src/python/pants/reporting:report', # TODO(pl): Bust this out ], ) @@ -179,7 +199,7 @@ python_library( name = 'workunit', sources = ['workunit.py'], dependencies = [ - '3rdparty/python:six', + '3rdparty/python:future', 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', 'src/python/pants/util:rwbuf', @@ -200,6 +220,7 @@ python_library( name='exiter', sources=['exiter.py'], dependencies=[ + '3rdparty/python:future', '3rdparty/python:faulthandler', 'src/python/pants/util:dirutil' ] diff --git a/src/python/pants/base/build_environment.py b/src/python/pants/base/build_environment.py index e3d0c4e9882..8512fa34588 100644 --- a/src/python/pants/base/build_environment.py +++ b/src/python/pants/base/build_environment.py @@ -8,6 +8,7 @@ import logging import os import sys +from builtins import str from pants.base.build_root import BuildRoot from pants.scm.scm import Scm diff --git a/src/python/pants/base/cmd_line_spec_parser.py b/src/python/pants/base/cmd_line_spec_parser.py index 2b35652c038..ed6bddcb992 100644 --- a/src/python/pants/base/cmd_line_spec_parser.py +++ b/src/python/pants/base/cmd_line_spec_parser.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import os +from builtins import object from pants.base.specs import DescendantAddresses, SiblingAddresses, SingleAddress diff --git a/src/python/pants/base/exiter.py b/src/python/pants/base/exiter.py index a0f6668c26f..2942e3f00eb 100644 --- a/src/python/pants/base/exiter.py +++ b/src/python/pants/base/exiter.py @@ -11,6 +11,7 @@ import signal import sys import traceback +from builtins import object import faulthandler import six diff --git a/src/python/pants/base/fingerprint_strategy.py b/src/python/pants/base/fingerprint_strategy.py index 787c3c86854..38e535c5c74 100644 --- a/src/python/pants/base/fingerprint_strategy.py +++ b/src/python/pants/base/fingerprint_strategy.py @@ -7,6 +7,7 @@ import logging from abc import abstractmethod +from builtins import object from pants.util.meta import AbstractClass diff --git a/src/python/pants/base/generator.py b/src/python/pants/base/generator.py index 9f77e8143a9..3a044b42de1 100644 --- a/src/python/pants/base/generator.py +++ b/src/python/pants/base/generator.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import pprint +from builtins import object import pystache diff --git a/src/python/pants/base/hash_utils.py b/src/python/pants/base/hash_utils.py index 1f7ca4a1f7c..929ecd6c44c 100644 --- a/src/python/pants/base/hash_utils.py +++ b/src/python/pants/base/hash_utils.py @@ -7,6 +7,7 @@ import hashlib import json +from builtins import object def hash_all(strs, digest=None): diff --git a/src/python/pants/base/mustache.py b/src/python/pants/base/mustache.py index c22e688df71..f4f40f2f8ca 100644 --- a/src/python/pants/base/mustache.py +++ b/src/python/pants/base/mustache.py @@ -7,6 +7,7 @@ import os import pkgutil +from builtins import object import pystache import six diff --git a/src/python/pants/base/parse_context.py b/src/python/pants/base/parse_context.py index 48fe9177639..8b39c712cd4 100644 --- a/src/python/pants/base/parse_context.py +++ b/src/python/pants/base/parse_context.py @@ -7,6 +7,7 @@ import functools import threading +from builtins import object class Storage(threading.local): diff --git a/src/python/pants/base/payload.py b/src/python/pants/base/payload.py index c2da0986719..50e3c175f63 100644 --- a/src/python/pants/base/payload.py +++ b/src/python/pants/base/payload.py @@ -5,6 +5,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +from builtins import object from hashlib import sha1 @@ -30,7 +31,7 @@ def __init__(self): @property def fields(self): - return self._fields.items() + return list(self._fields.items()) def as_dict(self): """Return the Payload object as a dict.""" diff --git a/src/python/pants/base/payload_field.py b/src/python/pants/base/payload_field.py index a026b1c7ed3..f211e662b3e 100644 --- a/src/python/pants/base/payload_field.py +++ b/src/python/pants/base/payload_field.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) from abc import abstractmethod +from builtins import object from hashlib import sha1 from twitter.common.collections import OrderedSet diff --git a/src/python/pants/base/revision.py b/src/python/pants/base/revision.py index 13c5b8b46f2..b6e492d244c 100644 --- a/src/python/pants/base/revision.py +++ b/src/python/pants/base/revision.py @@ -6,9 +6,13 @@ unicode_literals, with_statement) import re -from itertools import izip_longest +from builtins import map, object, str +from functools import total_ordering +from future.moves.itertools import zip_longest + +@total_ordering class Revision(object): """Represents a software revision that is comparable to another revision describing the same software. @@ -74,7 +78,7 @@ def lenient(cls, rev): """ rev = re.sub(r'(\d)([a-zA-Z])', r'\1.\2', rev) rev = re.sub(r'([a-zA-Z])(\d)', r'\1.\2', rev) - return cls(*map(cls._parse_atom, re.split(r'[.+_\-]', rev))) + return cls(*list(map(cls._parse_atom, re.split(r'[.+_\-]', rev)))) def __init__(self, *components): self._components = components @@ -87,21 +91,26 @@ def components(self): """ return list(self._components) - def __cmp__(self, other): - for ours, theirs in izip_longest(self._components, other._components, fillvalue=0): - difference = cmp(ours, theirs) - if difference != 0: - return difference - return 0 + def _is_valid_operand(self, other): + return hasattr(other, '_components') def __repr__(self): return '{}({})'.format(self.__class__.__name__, ', '.join(map(repr, self._components))) def __eq__(self, other): - return hasattr(other, '_components') and tuple(self._components) == tuple(other._components) - - def __ne__(self, other): - return not self.__eq__(other) + if not self._is_valid_operand(other): + return False # TODO(python3port): typically this should return NotImplemented. + # Returning False for now to avoid changing prior API. + return tuple(self._components) == tuple(other._components) + + def __lt__(self, other): + if not self._is_valid_operand(other): + return AttributeError # TODO(python3port): typically this should return NotImplemented. + # Returning AttributeError for now to avoid changing prior API. + for ours, theirs in zip_longest(self._components, other._components, fillvalue=0): + if ours != theirs: + return ours < theirs + return False def __hash__(self): return hash(self._components) diff --git a/src/python/pants/base/run_info.py b/src/python/pants/base/run_info.py index b17567a1b3e..3d39c46c561 100644 --- a/src/python/pants/base/run_info.py +++ b/src/python/pants/base/run_info.py @@ -10,6 +10,7 @@ import re import socket import time +from builtins import object, str from pants import version from pants.base.build_environment import get_buildroot, get_scm diff --git a/src/python/pants/base/worker_pool.py b/src/python/pants/base/worker_pool.py index b90688e71e2..35465891bac 100644 --- a/src/python/pants/base/worker_pool.py +++ b/src/python/pants/base/worker_pool.py @@ -8,10 +8,12 @@ import multiprocessing import signal import sys -import thread import threading +from builtins import next, object from multiprocessing.pool import ThreadPool +from future.moves import _thread + from pants.reporting.report import Report @@ -103,7 +105,7 @@ def error(e): # We filter out Nones defensively. There shouldn't be any, but if a bug causes one, # Pants might hang indefinitely without this filtering. - work_iter = iter(filter(None, work_chain)) + work_iter = (_f for _f in work_chain if _f) def submit_next(): try: @@ -149,7 +151,7 @@ def _do_work(self, func, args_tuple, workunit_name, workunit_parent, on_failure= except KeyboardInterrupt: # If a worker thread intercepts a KeyboardInterrupt, we want to propagate it to the main # thread. - thread.interrupt_main() + _thread.interrupt_main() raise except Exception as e: if on_failure: diff --git a/src/python/pants/base/workunit.py b/src/python/pants/base/workunit.py index 071ec1029c8..c9bc368fb62 100644 --- a/src/python/pants/base/workunit.py +++ b/src/python/pants/base/workunit.py @@ -9,10 +9,9 @@ import re import time import uuid +from builtins import object, range from collections import namedtuple -from six.moves import range - from pants.util.dirutil import safe_mkdir_for from pants.util.memo import memoized_method from pants.util.rwbuf import FileBackedRWBuf diff --git a/tests/python/pants_test/base/BUILD b/tests/python/pants_test/base/BUILD index c769bd803e1..8b1410e90e0 100644 --- a/tests/python/pants_test/base/BUILD +++ b/tests/python/pants_test/base/BUILD @@ -36,6 +36,7 @@ python_tests( name = 'exclude_target_regexp_integration', sources = [ 'test_exclude_target_regexp_integration.py' ], dependencies = [ + '3rdparty/python:future', 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', ], @@ -58,6 +59,7 @@ python_library( name = 'context_utils', sources = ['context_utils.py'], dependencies = [ + '3rdparty/python:future', '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/base:workunit', 'src/python/pants/build_graph', @@ -70,6 +72,7 @@ python_tests( name = 'deprecated', sources = ['test_deprecated.py'], dependencies = [ + '3rdparty/python:future', 'src/python/pants/base:deprecated', 'src/python/pants:version', ] @@ -109,6 +112,7 @@ python_tests( name = 'hash_utils', sources = ['test_hash_utils.py'], dependencies = [ + '3rdparty/python:future', 'src/python/pants/base:hash_utils', 'src/python/pants/util:contextutil', ] @@ -185,6 +189,7 @@ python_tests( name = 'worker_pool', sources = ['test_worker_pool.py'], dependencies = [ + '3rdparty/python:future', 'src/python/pants/base:worker_pool', 'src/python/pants/util:contextutil', ] diff --git a/tests/python/pants_test/base/context_utils.py b/tests/python/pants_test/base/context_utils.py index ae84f88569e..01995365067 100644 --- a/tests/python/pants_test/base/context_utils.py +++ b/tests/python/pants_test/base/context_utils.py @@ -7,6 +7,7 @@ import logging import sys +from builtins import map, object from contextlib import contextmanager from twitter.common.collections import maybe_list @@ -107,7 +108,7 @@ def subproc_map(self, f, items): :API: public """ # Just execute in-process. - return map(f, items) + return list(map(f, items)) def create_context_from_options(options, target_roots=None, build_graph=None, diff --git a/tests/python/pants_test/base/test_deprecated.py b/tests/python/pants_test/base/test_deprecated.py index 99f56250ecb..337528a6105 100644 --- a/tests/python/pants_test/base/test_deprecated.py +++ b/tests/python/pants_test/base/test_deprecated.py @@ -7,6 +7,7 @@ import unittest import warnings +from builtins import object, str from contextlib import contextmanager from pants.base.deprecated import (BadDecoratorNestingError, BadRemovalVersionError, diff --git a/tests/python/pants_test/base/test_exclude_target_regexp_integration.py b/tests/python/pants_test/base/test_exclude_target_regexp_integration.py index 62da96793a5..f2cdfb9eb6d 100644 --- a/tests/python/pants_test/base/test_exclude_target_regexp_integration.py +++ b/tests/python/pants_test/base/test_exclude_target_regexp_integration.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import os +from builtins import object, zip from contextlib import contextmanager from pants.base.build_environment import get_buildroot diff --git a/tests/python/pants_test/base/test_hash_utils.py b/tests/python/pants_test/base/test_hash_utils.py index fa0222e0dfa..c5e0249244f 100644 --- a/tests/python/pants_test/base/test_hash_utils.py +++ b/tests/python/pants_test/base/test_hash_utils.py @@ -8,6 +8,7 @@ import hashlib import math import unittest +from builtins import range, str from pants.base.hash_utils import Sharder, hash_all, hash_file from pants.util.contextutil import temporary_file diff --git a/tests/python/pants_test/base/test_worker_pool.py b/tests/python/pants_test/base/test_worker_pool.py index 9fee94ee880..ec003c7e5ec 100644 --- a/tests/python/pants_test/base/test_worker_pool.py +++ b/tests/python/pants_test/base/test_worker_pool.py @@ -7,6 +7,7 @@ import threading import unittest +from builtins import object from pants.base.worker_pool import Work, WorkerPool from pants.base.workunit import WorkUnit