diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 38d7b996c7c221..0a07bb29b34319 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -555,6 +555,30 @@ def test_workers_available_cores(self, compile_dir): self.assertTrue(compile_dir.called) self.assertEqual(compile_dir.call_args[-1]['workers'], None) + def test_deterministic_serialization(self): + # http://bugs.python.org/issue34722 + # We have to test this in subprocesses since the nondeterminism + # revolves around the hash seed. + with open(os.path.join(self.pkgdir, 'set.py'), 'w') as f: + f.write("def test(x):\n") + f.write(" if x in {'ONE', 'TWO', 'THREE'}:\n") + f.write(" pass\n") + + def compile_dir(): + self.assertRunOK( + '-q', + '--invalidation-mode=unchecked-hash', + self.pkgdir, + ) + for file in sorted(os.listdir(self.pkgdir_cachedir)): + with open(os.path.join(self.pkgdir_cachedir, file), 'rb') as f: + yield f.read() + + serialized = list(compile_dir()) + for _ in range(5): + shutil.rmtree(self.pkgdir_cachedir) + self.assertEqual(serialized, list(compile_dir())) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-09-21-15-38-47.bpo-34722.F6VE5R.rst b/Misc/NEWS.d/next/Core and Builtins/2018-09-21-15-38-47.bpo-34722.F6VE5R.rst new file mode 100644 index 00000000000000..509c052aa1fa60 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-09-21-15-38-47.bpo-34722.F6VE5R.rst @@ -0,0 +1,4 @@ +Ensures that sets / frozensets marshal in a consistent order by sorting the +serialised items before writing them. This will result in somewhat slower +serialisation but bytecode files using set literals in certain contexts will +now serialise deterministically where it didn't before. diff --git a/Python/marshal.c b/Python/marshal.c index 21cdd60c7e1377..0455c1f8ff9f81 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -496,7 +496,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object((PyObject *)NULL, p); } else if (PyAnySet_CheckExact(v)) { - PyObject *value, *it; + PyObject *value, *l; if (PyObject_TypeCheck(v, &PySet_Type)) W_TYPE(TYPE_SET, p); @@ -509,17 +509,27 @@ w_complex_object(PyObject *v, char flag, WFILE *p) return; } W_SIZE(n, p); - it = PyObject_GetIter(v); - if (it == NULL) { + l = PySequence_List(v); + if (l == NULL) { p->depth--; p->error = WFERR_UNMARSHALLABLE; return; } - while ((value = PyIter_Next(it)) != NULL) { - w_object(value, p); - Py_DECREF(value); + for (i = 0; i < n; i++) { + value = PyList_GetItem(l, i); + value = PyMarshal_WriteObjectToString(value, Py_MARSHAL_VERSION); + PyList_SetItem(l, i, value); + } + if (PyList_Sort(l) == -1) { + Py_DECREF(l); + p->depth--; + p->error = WFERR_UNMARSHALLABLE; + return; + } + for (i = 0; i < n; i++) { + w_object(PyList_GetItem(l, i), p); } - Py_DECREF(it); + Py_DECREF(l); if (PyErr_Occurred()) { p->depth--; p->error = WFERR_UNMARSHALLABLE;