From a84eb7c4e49a6ef63373b1cbcc07597b05f5e03c Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 19:15:25 -0700 Subject: [PATCH 01/12] [engine] fix unpickling error in multiprocessing engine tests --- src/python/pants/engine/exp/engine.py | 7 +------ src/python/pants/engine/exp/storage.py | 2 +- tests/python/pants_test/engine/exp/test_engine.py | 2 -- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/python/pants/engine/exp/engine.py b/src/python/pants/engine/exp/engine.py index f5f2c1b50c2..6a26b33a46d 100644 --- a/src/python/pants/engine/exp/engine.py +++ b/src/python/pants/engine/exp/engine.py @@ -5,6 +5,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import cPickle as pickle import functools import logging import multiprocessing @@ -21,12 +22,6 @@ from pants.util.objects import datatype -try: - import cPickle as pickle -except ImportError: - import pickle - - logger = logging.getLogger(__name__) diff --git a/src/python/pants/engine/exp/storage.py b/src/python/pants/engine/exp/storage.py index cdeb51bcd49..864d51ef39e 100644 --- a/src/python/pants/engine/exp/storage.py +++ b/src/python/pants/engine/exp/storage.py @@ -6,7 +6,7 @@ unicode_literals, with_statement) import cPickle as pickle -import cStringIO as StringIO +import StringIO import sys from abc import abstractmethod from binascii import hexlify diff --git a/tests/python/pants_test/engine/exp/test_engine.py b/tests/python/pants_test/engine/exp/test_engine.py index 2e1da98f043..2d211ecbfef 100644 --- a/tests/python/pants_test/engine/exp/test_engine.py +++ b/tests/python/pants_test/engine/exp/test_engine.py @@ -46,7 +46,6 @@ def test_serial_engine_simple(self): engine = LocalSerialEngine(self.scheduler, self.storage, self.cache) self.assert_engine(engine) - @unittest.skip('https://github.com/pantsbuild/pants/issues/3149') def test_multiprocess_engine_multi(self): with self.multiprocessing_engine() as engine: self.assert_engine(engine) @@ -62,7 +61,6 @@ def test_multiprocess_unpickleable(self): with self.assertRaises(SerializationError): engine.execute(build_request) - @unittest.skip('https://github.com/pantsbuild/pants/issues/3149') def test_rerun_with_cache(self): with self.multiprocessing_engine() as engine: self.assert_engine(engine) From 01e79f7a5eb2e52de8baf9f840bec5184dbeb64b Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:42:09 -0700 Subject: [PATCH 02/12] [engine] fix unpickling error in multiprocessing engine, Trigger 1 From e727ed70610553d441d3a392d872435c21c50cd3 Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:42:47 -0700 Subject: [PATCH 03/12] [engine] fix unpickling error in multiprocessing engine, Trigger 2 From 2755d6e9e3561a36cd9b42d45de7ee694d2246de Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:42:54 -0700 Subject: [PATCH 04/12] [engine] fix unpickling error in multiprocessing engine, Trigger 3 From 73cf812c24f850080e184ce0b58253c7f1c9e518 Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:43:01 -0700 Subject: [PATCH 05/12] [engine] fix unpickling error in multiprocessing engine, Trigger 4 From ed96cd14fc453011e3d14bda92f457bc7d76b061 Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:43:08 -0700 Subject: [PATCH 06/12] [engine] fix unpickling error in multiprocessing engine, Trigger 5 From c58bb5e970e2e6211adf5cbd27a124049de0701a Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:43:14 -0700 Subject: [PATCH 07/12] [engine] fix unpickling error in multiprocessing engine, Trigger 6 From 8f3896910552505b36d97512fd392e9b8258f1f2 Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:43:21 -0700 Subject: [PATCH 08/12] [engine] fix unpickling error in multiprocessing engine, Trigger 7 From e57a1a71ebaabdbcedba584d8f808b41b5bbe02e Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:43:27 -0700 Subject: [PATCH 09/12] [engine] fix unpickling error in multiprocessing engine, Trigger 8 From 53f736b21bb69db50c75828abe508b164ff71777 Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:43:34 -0700 Subject: [PATCH 10/12] [engine] fix unpickling error in multiprocessing engine, Trigger 9 From 8ee0ac360d68ec64fd2831323d5215c0bfa853d7 Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Fri, 22 Apr 2016 20:43:41 -0700 Subject: [PATCH 11/12] [engine] fix unpickling error in multiprocessing engine, Trigger 10 From 514d22c7323491cc5cb709087307056ef0b9c9e6 Mon Sep 17 00:00:00 2001 From: Peiyu Wang Date: Mon, 25 Apr 2016 12:44:02 -0700 Subject: [PATCH 12/12] appears to hit a bug in 2.7.9 --- src/python/pants/engine/exp/storage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/python/pants/engine/exp/storage.py b/src/python/pants/engine/exp/storage.py index 864d51ef39e..6cecb7caaa0 100644 --- a/src/python/pants/engine/exp/storage.py +++ b/src/python/pants/engine/exp/storage.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import cPickle as pickle -import StringIO import sys from abc import abstractmethod from binascii import hexlify @@ -14,6 +13,8 @@ from contextlib import closing from functools import total_ordering from hashlib import sha1 +from io import BytesIO +from StringIO import StringIO from struct import Struct as StdlibStruct import lmdb @@ -166,7 +167,7 @@ def put(self, obj): Longer term see https://github.com/pantsbuild/pants/issues/2969 """ try: - with closing(StringIO.StringIO()) as buf: + with closing(BytesIO()) as buf: pickler = pickle.Pickler(buf, protocol=self._protocol) pickler.fast = 1 pickler.dump(obj) @@ -493,7 +494,7 @@ def get(self, key): with self._env.begin(db=self._db, buffers=True) as txn: value = txn.get(key) if value is not None: - return StringIO.StringIO(value) + return StringIO(value) return None def put(self, key, value):