python
diff --git a/‎Lib/test/test_marshal.py
Lines changed: 22 additions & 11 deletions b/‎Lib/test/test_marshal.py
Lines changed: 22 additions & 11 deletions
diff --git a/‎Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst
Lines changed: 1 addition & 1 deletion b/‎Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst
Lines changed: 1 addition & 1 deletion
diff --git a/‎Python/marshal.c
Lines changed: 8 additions & 8 deletions b/‎Python/marshal.c
Lines changed: 8 additions & 8 deletions
@@ -345,18 +345,29 @@ def test_eof(self):
             self.assertRaises(EOFError, marshal.loads, data[0: i])
 
     def test_deterministic_sets(self):
-        # bpo-37596: sets and frozensets must always marshal deterministically!
-        # Test this by seeding string hashes:
-        elements = "(float('nan'), b'a', b'b', b'c', 'x', 'y', 'z')"
+        # bpo-37596: To support reproducible builds, sets and frozensets need to
+        # have their elements serialized in a consistent order (even when they
+        # have been scrambled by hash randomization):
         for kind in ("set", "frozenset"):
-            script = f"import marshal; print(marshal.dumps({kind}({elements})))"
-            with self.subTest(kind):
-                # {nan, b'b', 'x', b'a', b'c', 'y', 'z'}
-                _, a, _ = assert_python_ok("-c", script, PYTHONHASHSEED="0")
-                # {nan, 'x', b'a', 'z', b'b', 'y', b'c'}
-                _, b, _ = assert_python_ok("-c", script, PYTHONHASHSEED="1")
-                self.assertEqual(a, b)
-
+            for elements in (
+                "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'",
+                # Also test for bad interactions with backreferencing:
+                "('string', 1), ('string', 2), ('string', 3)",
+            ):
+                s = f"{kind}([{elements}])"
+                with self.subTest(s):
+                    # First, make sure that our test case still has different
+                    # orders under hash seeds 0 and 1. If this check fails, we
+                    # need to update this test with different elements:
+                    args = ["-c", f"print({s})"]
+                    _, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
+                    _, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
+                    self.assertNotEqual(repr_0, repr_1)
+                    # Then, perform the actual test:
+                    args = ["-c", f"import marshal; print(marshal.dumps({s}))"]
+                    _, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
+                    _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
+                    self.assertEqual(dump_0, dump_1)
 
 LARGE_SIZE = 2**31
 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
 
@@ -1,2 +1,2 @@
 Ensure that :class:`set` and :class:`frozenset` objects are always
-:mod:`marshalled <marshal>` deterministically.
+:mod:`marshalled <marshal>` reproducibly.
@@ -503,10 +503,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
             W_TYPE(TYPE_SET, p);
         n = PySet_GET_SIZE(v);
         W_SIZE(n, p);
-        // bpo-37596: We need to write the elements in a deterministic order!
-        // Since we know that they all need marshal support anyways, this can
-        // be conveniently accomplished by using their marshal serializations
-        // as sort keys:
+        // bpo-37596: To support reproducible builds, sets and frozensets need
+        // to have their elements serialized in a consistent order (even when
+        // they have been scrambled by hash randomization). To ensure this, we
+        // use an order equivalent to sorted(v, key=marshal.dumps):
         PyObject *pairs = PyList_New(0);
         if (pairs == NULL) {
             p->error = WFERR_NOMEMORY;
@@ -520,15 +520,15 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
             }
             PyObject *pair = PyTuple_Pack(2, dump, value);
             Py_DECREF(dump);
-            int error = pair == NULL || PyList_Append(pairs, pair);
-            Py_XDECREF(pair);
-            if (error) {
+            if (pair == NULL || PyList_Append(pairs, pair)) {
                 p->error = WFERR_NOMEMORY;
+                Py_XDECREF(pair);
                 goto anyset_done;
             }
+            Py_DECREF(pair);
         }
         if (PyList_Sort(pairs)) {
-            p->error = WFERR_UNMARSHALLABLE;
+            p->error = WFERR_NOMEMORY;
             goto anyset_done;
         }
         for (Py_ssize_t i = 0; i < n; i++) {
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	Ensure that :class:`set` and :class:`frozenset` objects are always
`2`		-:mod:`marshalled <marshal>` deterministically.
	`2`	+:mod:`marshalled <marshal>` reproducibly.