Add future lib and port src/base (#6067)

### Description
Implements stage 2 of [porting plan](#6062), which is adding the future library.

Also ports the `src/pants/base` and `tests/python/pants_test/base` as a demo of how this tool works.

At this stage, we aren't testing things yet with the Python 3 interpreter, even though technically it's Python 3 code. The goal for now is to make sure things still work on Python 2. 

#### FYI - how the backports work
The `future` library adds several modules: `future`, `builtins`, `past`, and individual ports like `queue` (instead of `Queue`). 

These added names all correspond exactly to Python 3 code. When running Python 3, it will simply use the native Python 3 version. When using Python 2, it will use the library implementation, which is semantically equivalent to Python 3.

#### FYI - using old `moves` interface
Typically `futurize` will try to use this idiom for certain backports

```python
from future import standard_library
standard_library.install_aliases()

import [backports]
```

This however does not work in our codebase, because all imports must appear above code with iSort, so the `install_aliases()` line gets moved below the relevant imports. 

Instead, we are using the legacy `future.moves` library. Once we remove Python 2 support we'll want to convert this type of import `from future.moves.itertools import x` to `from itertools import x`.

3rdparty/python/requirements.txt

@@ -1,11 +1,13 @@
 ansicolors==1.0.2
 beautifulsoup4>=4.3.2,<4.4
 cffi==1.11.1
+configparser==3.5.0
 contextlib2==0.5.5
 coverage>=4.5,<4.6
 docutils>=0.12,<0.13
 fasteners==0.14.1
 faulthandler==2.6
+future==0.16.0
 futures==3.0.5
 isort==4.2.5
 Markdown==2.1.1

pants-plugins/src/python/internal_backend/sitegen/tasks/sitegen.py

@@ -5,6 +5,7 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import cgi
 import collections
 import json
 import os
@@ -354,7 +355,9 @@ def generate_generated(config, here):
 
 def render_html(dst, config, soups, precomputed, template):
   soup = soups[dst]
-  renderer = Renderer()
+  renderer = Renderer(escape=cgi.escape)  # TODO(python3port): cgi.escape is deprecated in favor html.escape
+                                          # (the default used by Pystache on Python3). html.escape() escapes single
+                                          # quotes, whereas cgi.escape() does not. This will need to be addressed.
   title = precomputed.page[dst].title
   topdots = ('../' * dst.count('/'))
   if soup.body:

src/python/pants/base/BUILD

@@ -5,6 +5,7 @@ python_library(
   name = 'build_environment',
   sources = ['build_environment.py'],
   dependencies = [
+    '3rdparty/python:future',
     ':build_root',
     'src/python/pants/scm',
     'src/python/pants/scm:git',
@@ -75,6 +76,7 @@ python_library(
   name = 'fingerprint_strategy',
   sources = ['fingerprint_strategy.py'],
   dependencies = [
+    '3rdparty/python:future',
     ':deprecated',
     'src/python/pants/util:meta',
   ]
@@ -85,19 +87,24 @@ python_library(
   sources = ['generator.py'],
   dependencies = [
     ':mustache',
+    '3rdparty/python:future',
     '3rdparty/python:pystache',
   ]
 )
 
 python_library(
   name = 'hash_utils',
   sources = ['hash_utils.py'],
+  dependencies = [
+    '3rdparty/python:future',
+  ]
 )
 
 python_library(
   name = 'mustache',
   sources = ['mustache.py'],
   dependencies = [
+    '3rdparty/python:future',
     '3rdparty/python:pystache',
     '3rdparty/python:six',
   ]
@@ -106,17 +113,24 @@ python_library(
 python_library(
   name = 'parse_context',
   sources = ['parse_context.py'],
+  dependencies = [
+    '3rdparty/python:future',
+  ]
 )
 
 python_library(
   name = 'payload',
   sources = ['payload.py'],
+  dependencies = [
+    '3rdparty/python:future',
+  ]
 )
 
 python_library(
   name = 'payload_field',
   sources = ['payload_field.py'],
   dependencies = [
+    '3rdparty/python:future',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     ':deprecated',
     ':hash_utils',
@@ -136,12 +150,16 @@ python_library(
 python_library(
   name = 'revision',
   sources = ['revision.py'],
+  dependencies = [
+    '3rdparty/python:future',
+  ]
 )
 
 python_library(
   name = 'run_info',
   sources = ['run_info.py'],
   dependencies = [
+    '3rdparty/python:future',
     ':build_environment',
     'src/python/pants/util:dirutil',
     'src/python/pants:version',
@@ -152,6 +170,7 @@ python_library(
   name = 'cmd_line_spec_parser',
   sources = ['cmd_line_spec_parser.py'],
   dependencies = [
+    '3rdparty/python:future',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     ':build_file',
     ':specs',
@@ -171,6 +190,7 @@ python_library(
   name = 'worker_pool',
   sources = ['worker_pool.py'],
   dependencies = [
+    '3rdparty/python:future',
     'src/python/pants/reporting:report', # TODO(pl): Bust this out
   ],
 )
@@ -179,7 +199,7 @@ python_library(
   name = 'workunit',
   sources = ['workunit.py'],
   dependencies = [
-    '3rdparty/python:six',
+    '3rdparty/python:future',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:memo',
     'src/python/pants/util:rwbuf',
@@ -200,6 +220,7 @@ python_library(
   name='exiter',
   sources=['exiter.py'],
   dependencies=[
+    '3rdparty/python:future',
     '3rdparty/python:faulthandler',
     'src/python/pants/util:dirutil'
   ]

src/python/pants/base/build_environment.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import sys
+from builtins import str
 
 from pants.base.build_root import BuildRoot
 from pants.scm.scm import Scm

src/python/pants/base/cmd_line_spec_parser.py

@@ -6,6 +6,7 @@
                         unicode_literals, with_statement)
 
 import os
+from builtins import object
 
 from pants.base.specs import DescendantAddresses, SiblingAddresses, SingleAddress
 

src/python/pants/base/exiter.py

@@ -11,6 +11,7 @@
 import signal
 import sys
 import traceback
+from builtins import object
 
 import faulthandler
 import six

src/python/pants/base/fingerprint_strategy.py

@@ -7,6 +7,7 @@
 
 import logging
 from abc import abstractmethod
+from builtins import object
 
 from pants.util.meta import AbstractClass
 

src/python/pants/base/generator.py

@@ -6,6 +6,7 @@
                         unicode_literals, with_statement)
 
 import pprint
+from builtins import object
 
 import pystache
 

src/python/pants/base/hash_utils.py

@@ -7,6 +7,7 @@
 
 import hashlib
 import json
+from builtins import object
 
 
 def hash_all(strs, digest=None):

src/python/pants/base/mustache.py

@@ -7,6 +7,7 @@
 
 import os
 import pkgutil
+from builtins import object
 
 import pystache
 import six

src/python/pants/base/parse_context.py

@@ -7,6 +7,7 @@
 
 import functools
 import threading
+from builtins import object
 
 
 class Storage(threading.local):

src/python/pants/base/payload.py

@@ -5,6 +5,7 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+from builtins import object
 from hashlib import sha1
 
 
@@ -30,7 +31,7 @@ def __init__(self):
 
   @property
   def fields(self):
-    return self._fields.items()
+    return list(self._fields.items())
 
   def as_dict(self):
     """Return the Payload object as a dict."""

src/python/pants/base/payload_field.py

@@ -6,6 +6,7 @@
                         unicode_literals, with_statement)
 
 from abc import abstractmethod
+from builtins import object
 from hashlib import sha1
 
 from twitter.common.collections import OrderedSet

src/python/pants/base/revision.py

@@ -6,9 +6,13 @@
                         unicode_literals, with_statement)
 
 import re
-from itertools import izip_longest
+from builtins import map, object, str
+from functools import total_ordering
 
+from future.moves.itertools import zip_longest
 
+
+@total_ordering
 class Revision(object):
   """Represents a software revision that is comparable to another revision describing the same
   software.
@@ -74,7 +78,7 @@ def lenient(cls, rev):
     """
     rev = re.sub(r'(\d)([a-zA-Z])', r'\1.\2', rev)
     rev = re.sub(r'([a-zA-Z])(\d)', r'\1.\2', rev)
-    return cls(*map(cls._parse_atom, re.split(r'[.+_\-]', rev)))
+    return cls(*list(map(cls._parse_atom, re.split(r'[.+_\-]', rev))))
 
   def __init__(self, *components):
     self._components = components
@@ -87,21 +91,26 @@ def components(self):
     """
     return list(self._components)
 
-  def __cmp__(self, other):
-    for ours, theirs in izip_longest(self._components, other._components, fillvalue=0):
-      difference = cmp(ours, theirs)
-      if difference != 0:
-        return difference
-    return 0
+  def _is_valid_operand(self, other):
+    return hasattr(other, '_components')
 
   def __repr__(self):
     return '{}({})'.format(self.__class__.__name__, ', '.join(map(repr, self._components)))
 
   def __eq__(self, other):
-    return hasattr(other, '_components') and tuple(self._components) == tuple(other._components)
-
-  def __ne__(self, other):
-    return not self.__eq__(other)
+    if not self._is_valid_operand(other):
+      return False  # TODO(python3port): typically this should return NotImplemented.
+                    # Returning False for now to avoid changing prior API.
+    return tuple(self._components) == tuple(other._components)
+
+  def __lt__(self, other):
+    if not self._is_valid_operand(other):
+      return AttributeError  # TODO(python3port): typically this should return NotImplemented.
+                             # Returning AttributeError for now to avoid changing prior API.
+    for ours, theirs in zip_longest(self._components, other._components, fillvalue=0):
+      if ours != theirs:
+        return ours < theirs
+    return False
 
   def __hash__(self):
     return hash(self._components)

src/python/pants/base/run_info.py

@@ -10,6 +10,7 @@
 import re
 import socket
 import time
+from builtins import object, str
 
 from pants import version
 from pants.base.build_environment import get_buildroot, get_scm

src/python/pants/base/worker_pool.py

@@ -8,10 +8,12 @@
 import multiprocessing
 import signal
 import sys
-import thread
 import threading
+from builtins import next, object
 from multiprocessing.pool import ThreadPool
 
+from future.moves import _thread
+
 from pants.reporting.report import Report
 
 
@@ -103,7 +105,7 @@ def error(e):
 
     # We filter out Nones defensively. There shouldn't be any, but if a bug causes one,
     # Pants might hang indefinitely without this filtering.
-    work_iter = iter(filter(None, work_chain))
+    work_iter = (_f for _f in work_chain if _f)
 
     def submit_next():
       try:
@@ -149,7 +151,7 @@ def _do_work(self, func, args_tuple, workunit_name, workunit_parent, on_failure=
     except KeyboardInterrupt:
       # If a worker thread intercepts a KeyboardInterrupt, we want to propagate it to the main
       # thread.
-      thread.interrupt_main()
+      _thread.interrupt_main()
       raise
     except Exception as e:
       if on_failure:

src/python/pants/base/workunit.py

@@ -9,10 +9,9 @@
 import re
 import time
 import uuid
+from builtins import object, range
 from collections import namedtuple
 
-from six.moves import range
-
 from pants.util.dirutil import safe_mkdir_for
 from pants.util.memo import memoized_method
 from pants.util.rwbuf import FileBackedRWBuf

tests/python/pants_test/base/BUILD

@@ -36,6 +36,7 @@ python_tests(
   name = 'exclude_target_regexp_integration',
   sources = [ 'test_exclude_target_regexp_integration.py' ],
   dependencies = [
+    '3rdparty/python:future',
     'src/python/pants/util:process_handler',
     'tests/python/pants_test:int-test',
   ],
@@ -58,6 +59,7 @@ python_library(
   name = 'context_utils',
   sources = ['context_utils.py'],
   dependencies = [
+    '3rdparty/python:future',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/base:workunit',
     'src/python/pants/build_graph',
@@ -70,6 +72,7 @@ python_tests(
   name = 'deprecated',
   sources = ['test_deprecated.py'],
   dependencies = [
+    '3rdparty/python:future',
     'src/python/pants/base:deprecated',
     'src/python/pants:version',
   ]
@@ -109,6 +112,7 @@ python_tests(
   name = 'hash_utils',
   sources = ['test_hash_utils.py'],
   dependencies = [
+    '3rdparty/python:future',
     'src/python/pants/base:hash_utils',
     'src/python/pants/util:contextutil',
   ]
@@ -185,6 +189,7 @@ python_tests(
   name = 'worker_pool',
   sources = ['test_worker_pool.py'],
   dependencies = [
+    '3rdparty/python:future',
     'src/python/pants/base:worker_pool',
     'src/python/pants/util:contextutil',
   ]