bpo-44342: [Enum] changed pickling from by-value to by-name (GH-26658) (GH-26660)

by-value lookups could fail on complex enums, necessitating a check for
__reduce__ and possibly sabotaging the final enum;

by-name lookups should never fail, and sabotaging is no longer necessary
for class-based enum creation.
(cherry picked from commit 62f1d2b)

Co-authored-by: Ethan Furman <[email protected]>

Co-authored-by: Ethan Furman <[email protected]>

Author: miss-islington

Lib/enum.py

@@ -456,23 +456,6 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
         classdict['_all_bits_'] = 2 ** ((flag_mask).bit_length()) - 1
         classdict['_inverted_'] = None
         #
-        # If a custom type is mixed into the Enum, and it does not know how
-        # to pickle itself, pickle.dumps will succeed but pickle.loads will
-        # fail.  Rather than have the error show up later and possibly far
-        # from the source, sabotage the pickle protocol for this class so
-        # that pickle.dumps also fails.
-        #
-        # However, if the new class implements its own __reduce_ex__, do not
-        # sabotage -- it's on them to make sure it works correctly.  We use
-        # __reduce_ex__ instead of any of the others as it is preferred by
-        # pickle over __reduce__, and it handles all pickle protocols.
-        if '__reduce_ex__' not in classdict:
-            if member_type is not object:
-                methods = ('__getnewargs_ex__', '__getnewargs__',
-                        '__reduce_ex__', '__reduce__')
-                if not any(m in member_type.__dict__ for m in methods):
-                    _make_class_unpicklable(classdict)
-        #
         # create a default docstring if one has not been provided
         if '__doc__' not in classdict:
             classdict['__doc__'] = 'An enumeration.'
@@ -792,7 +775,7 @@ def _convert_(cls, name, module, filter, source=None, *, boundary=None):
         body['__module__'] = module
         tmp_cls = type(name, (object, ), body)
         cls = _simple_enum(etype=cls, boundary=boundary or KEEP)(tmp_cls)
-        cls.__reduce_ex__ = _reduce_ex_by_name
+        cls.__reduce_ex__ = _reduce_ex_by_global_name
         global_enum(cls)
         module_globals[name] = cls
         return cls
@@ -1030,7 +1013,7 @@ def __hash__(self):
         return hash(self._name_)
 
     def __reduce_ex__(self, proto):
-        return self.__class__, (self._value_, )
+        return getattr, (self.__class__, self._name_)
 
     # enum.property is used to provide access to the `name` and
     # `value` attributes of enum members while keeping some measure of
@@ -1091,7 +1074,7 @@ def _generate_next_value_(name, start, count, last_values):
         return name.lower()
 
 
-def _reduce_ex_by_name(self, proto):
+def _reduce_ex_by_global_name(self, proto):
     return self.name
 
 class FlagBoundary(StrEnum):
@@ -1795,6 +1778,6 @@ def _old_convert_(etype, name, module, filter, source=None, *, boundary=None):
         # unless some values aren't comparable, in which case sort by name
         members.sort(key=lambda t: t[0])
     cls = etype(name, members, module=module, boundary=boundary or KEEP)
-    cls.__reduce_ex__ = _reduce_ex_by_name
+    cls.__reduce_ex__ = _reduce_ex_by_global_name
     cls.__repr__ = global_enum_repr
     return cls

Lib/test/test_enum.py

@@ -830,7 +830,7 @@ def test_pickle_by_name(self):
         class ReplaceGlobalInt(IntEnum):
             ONE = 1
             TWO = 2
-        ReplaceGlobalInt.__reduce_ex__ = enum._reduce_ex_by_name
+        ReplaceGlobalInt.__reduce_ex__ = enum._reduce_ex_by_global_name
         for proto in range(HIGHEST_PROTOCOL):
             self.assertEqual(ReplaceGlobalInt.TWO.__reduce_ex__(proto), 'TWO')
 
@@ -1527,10 +1527,10 @@ class NEI(NamedInt, Enum):
         NI5 = NamedInt('test', 5)
         self.assertEqual(NI5, 5)
         self.assertEqual(NEI.y.value, 2)
-        test_pickle_exception(self.assertRaises, TypeError, NEI.x)
-        test_pickle_exception(self.assertRaises, PicklingError, NEI)
+        test_pickle_dump_load(self.assertIs, NEI.y)
+        test_pickle_dump_load(self.assertIs, NEI)
 
-    def test_subclasses_without_direct_pickle_support_using_name(self):
+    def test_subclasses_with_direct_pickle_support(self):
         class NamedInt(int):
             __qualname__ = 'NamedInt'
             def __new__(cls, *args):

Misc/NEWS.d/next/Library/2021-06-10-15-06-47.bpo-44342.qqkGlj.rst

@@ -0,0 +1 @@
+[Enum] Change pickling from by-value to by-name.