Skip to content

Commit

Permalink
Kickstart support drop for Python < 3.7
Browse files Browse the repository at this point in the history 
Step 1 of code clean-up: mechanically remove if-else branches that would never
run in Python/PyPy >= 3.7.  There's still some unused or redundant code, left
for the next step to facilitate review.

Note: all the remaining `sys.hexversion` tests were standardized to use
numerical comparison and hexadecimal integer literals (e.g. `0x30b00a7` for
version 3.11.0a7) as most of them already were.
  • Loading branch information
@leogama
leogama committed Jun 4, 2022
1 parent ff1969e commit fc46578
Show file tree
Hide file tree
Showing 15 changed files with 292 additions and 835 deletions.
654 changes: 183 additions & 471 deletions dill/_dill.py
View file

Large diffs are not rendered by default.

108 changes: 26 additions & 82 deletions dill/_objects.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,9 @@
# helper imports
import warnings; warnings.filterwarnings("ignore", category=DeprecationWarning)
import sys
PY3 = (hex(sys.hexversion) >= '0x30000f0')
if PY3:
import queue as Queue
import dbm as anydbm
else:
import Queue
import anydbm
import sets # deprecated/removed
import mutex # removed
try:
from cStringIO import StringIO # has StringI and StringO types
except ImportError: # only has StringIO type
if PY3:
from io import BytesIO as StringIO
else:
from StringIO import StringIO
import queue as Queue
import dbm as anydbm
from io import BytesIO as StringIO
import re
import array
import collections
Expand Down Expand Up @@ -64,8 +51,7 @@
try:
import bz2
import sqlite3
if PY3: import dbm.ndbm as dbm
else: import dbm
import dbm.ndbm as dbm
HAS_ALL = True
except ImportError: # Ubuntu
HAS_ALL = False
Expand Down Expand Up @@ -157,12 +143,8 @@ class _Struct(ctypes.Structure):
a['StringType'] = _str = str(1)
a['TupleType'] = _tuple = ()
a['TypeType'] = type
if PY3:
a['LongType'] = _int
a['UnicodeType'] = _str
else:
a['LongType'] = long(1)
a['UnicodeType'] = unicode(1)
a['LongType'] = _int
a['UnicodeType'] = _str
# built-in constants (CH 4)
a['CopyrightType'] = copyright
# built-in types (CH 5)
Expand All @@ -181,10 +163,6 @@ class _Struct(ctypes.Structure):
a['TZInfoType'] = datetime.tzinfo()
a['DateTimeType'] = datetime.datetime.today()
a['CalendarType'] = calendar.Calendar()
if not PY3:
a['SetsType'] = sets.Set()
a['ImmutableSetType'] = sets.ImmutableSet()
a['MutexType'] = mutex.mutex()
# numeric and mathematical types (CH 9)
a['DecimalType'] = decimal.Decimal(1)
a['CountType'] = itertools.count(0)
Expand Down Expand Up @@ -288,32 +266,22 @@ class _Struct(ctypes.Structure):
except ImportError:
pass
# other (concrete) object types
if PY3:
d['CellType'] = (_lambda)(0).__closure__[0]
a['XRangeType'] = _xrange = range(1)
else:
d['CellType'] = (_lambda)(0).func_closure[0]
a['XRangeType'] = _xrange = xrange(1)
d['CellType'] = (_lambda)(0).__closure__[0]
a['XRangeType'] = _xrange = range(1)
if not IS_PYPY:
d['MethodDescriptorType'] = type.__dict__['mro']
d['WrapperDescriptorType'] = type.__repr__
a['WrapperDescriptorType2'] = type.__dict__['__module__']
d['ClassMethodDescriptorType'] = type.__dict__['__prepare__' if PY3 else 'mro']
d['ClassMethodDescriptorType'] = type.__dict__['__prepare__']
# built-in functions (CH 2)
if PY3 or IS_PYPY:
_methodwrap = (1).__lt__
else:
_methodwrap = (1).__cmp__
_methodwrap = (1).__lt__
d['MethodWrapperType'] = _methodwrap
a['StaticMethodType'] = staticmethod(_method)
a['ClassMethodType'] = classmethod(_method)
a['PropertyType'] = property()
d['SuperType'] = super(Exception, _exception)
# string services (CH 7)
if PY3:
_in = _bytes
else:
_in = _str
_in = _bytes
a['InputType'] = _cstrI = StringIO(_in)
a['OutputType'] = _cstrO = StringIO()
# data types (CH 8)
Expand All @@ -328,25 +296,20 @@ class _Struct(ctypes.Structure):
a['QueueType'] = Queue.Queue()
# numeric and mathematical types (CH 9)
d['PartialType'] = functools.partial(int,base=2)
if PY3:
a['IzipType'] = zip('0','1')
else:
a['IzipType'] = itertools.izip('0','1')
a['IzipType'] = zip('0','1')
a['ChainType'] = itertools.chain('0','1')
d['ItemGetterType'] = operator.itemgetter(0)
d['AttrGetterType'] = operator.attrgetter('__repr__')
# file and directory access (CH 10)
if PY3: _fileW = _cstrO
else: _fileW = _tmpf
_fileW = _cstrO
# data persistence (CH 11)
if HAS_ALL:
a['ConnectionType'] = _conn = sqlite3.connect(':memory:')
a['CursorType'] = _conn.cursor()
a['ShelveType'] = shelve.Shelf({})
# data compression and archiving (CH 12)
if HAS_ALL:
if (hex(sys.hexversion) < '0x2070ef0') or PY3:
a['BZ2FileType'] = bz2.BZ2File(os.devnull) #FIXME: fail >= 3.3, 2.7.14
a['BZ2FileType'] = bz2.BZ2File(os.devnull) #FIXME: fail >= 3.3, 2.7.14
a['BZ2CompressorType'] = bz2.BZ2Compressor()
a['BZ2DecompressorType'] = bz2.BZ2Decompressor()
#a['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') #FIXME: fail >= 3.2
Expand All @@ -363,17 +326,10 @@ class _Struct(ctypes.Structure):
a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #FIXME: fail >= 3.2 and <= 2.6
#a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..."
# interprocess communication (CH 17)
if PY3:
a['SocketType'] = _socket = socket.socket() #FIXME: fail >= 3.3
a['SocketPairType'] = socket.socketpair()[0] #FIXME: fail >= 3.3
else:
a['SocketType'] = _socket = socket.socket()
a['SocketPairType'] = _socket._sock
a['SocketType'] = _socket = socket.socket() #FIXME: fail >= 3.3
a['SocketPairType'] = socket.socketpair()[0] #FIXME: fail >= 3.3
# python runtime services (CH 27)
if PY3:
a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1])
else:
a['GeneratorContextManagerType'] = contextlib.GeneratorContextManager(max)
a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1])

try: # ipython
__IPYTHON__ is True # is ipython
Expand Down Expand Up @@ -454,14 +410,9 @@ class _Struct(ctypes.Structure):
# built-in functions (CH 2)
x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty
# built-in types (CH 5)
if PY3:
x['DictionaryItemIteratorType'] = iter(type.__dict__.items())
x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys())
x['DictionaryValueIteratorType'] = iter(type.__dict__.values())
else:
x['DictionaryItemIteratorType'] = type.__dict__.iteritems()
x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys()
x['DictionaryValueIteratorType'] = type.__dict__.itervalues()
x['DictionaryItemIteratorType'] = iter(type.__dict__.items())
x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys())
x['DictionaryValueIteratorType'] = iter(type.__dict__.values())
# string services (CH 7)
x['StructType'] = struct.Struct('c')
x['CallableIteratorType'] = _srepattern.finditer('')
Expand All @@ -484,7 +435,7 @@ class _Struct(ctypes.Structure):
x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{})
# cryptographic services (CH 14)
x['HashType'] = hashlib.md5()
if (hex(sys.hexversion) < '0x30800a1'):
if (sys.hexversion < 0x30800a1):
x['HMACType'] = hmac.new(_in)
else:
x['HMACType'] = hmac.new(_in, digestmod='md5')
Expand Down Expand Up @@ -524,14 +475,9 @@ class _Struct(ctypes.Structure):
# built-in types (CH 5)
x['MemoryType'] = memoryview(_in) # 2.7
x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7
if PY3:
x['DictItemsType'] = _dict.items() # 2.7
x['DictKeysType'] = _dict.keys() # 2.7
x['DictValuesType'] = _dict.values() # 2.7
else:
x['DictItemsType'] = _dict.viewitems() # 2.7
x['DictKeysType'] = _dict.viewkeys() # 2.7
x['DictValuesType'] = _dict.viewvalues() # 2.7
x['DictItemsType'] = _dict.items() # 2.7
x['DictKeysType'] = _dict.keys() # 2.7
x['DictValuesType'] = _dict.values() # 2.7
# generic operating system services (CH 15)
x['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG')
x['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG')
Expand All @@ -543,10 +489,8 @@ class _Struct(ctypes.Structure):
x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2
except AttributeError:
pass
if PY3: # oddities: removed, etc
x['BufferType'] = x['MemoryType']
else:
x['BufferType'] = buffer('')
# oddities: removed, etc
x['BufferType'] = x['MemoryType']

# -- cleanup ----------------------------------------------------------------
a.update(d) # registered also succeed
Expand Down
106 changes: 0 additions & 106 deletions dill/_shims.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,111 +156,5 @@ def decorator(func):

_CELL_EMPTY = Getattr(_dill, '_CELL_EMPTY', None)

if _dill.OLD37:
if _dill.HAS_CTYPES and hasattr(_dill.ctypes, 'pythonapi') and hasattr(_dill.ctypes.pythonapi, 'PyCell_Set'):
# CPython
ctypes = _dill.ctypes

_PyCell_Set = ctypes.pythonapi.PyCell_Set

@move_to(_dill)
def _setattr(object, name, value):
if type(object) is _dill.CellType and name == 'cell_contents':
_PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object)
_PyCell_Set(object, value)
else:
setattr(object, name, value)

@move_to(_dill)
def _delattr(object, name):
if type(object) is _dill.CellType and name == 'cell_contents':
_PyCell_Set.argtypes = (ctypes.py_object, ctypes.c_void_p)
_PyCell_Set(object, None)
else:
delattr(object, name)

# General Python (not CPython) up to 3.6 is in a weird case, where it is
# possible to pickle recursive cells, but we can't assign directly to the
# cell.
elif _dill.PY3:
# Use nonlocal variables to reassign the cell value.
# https://stackoverflow.com/a/59276835
__nonlocal = ('nonlocal cell',)
exec('''def _setattr(cell, name, value):
if type(cell) is _dill.CellType and name == 'cell_contents':
def cell_setter(value):
%s
cell = value # pylint: disable=unused-variable
func = _dill.FunctionType(cell_setter.__code__, globals(), "", None, (cell,)) # same as cell_setter, but with cell being the cell's contents
func(value)
else:
setattr(cell, name, value)''' % __nonlocal)
move_to(_dill)(_setattr)

exec('''def _delattr(cell, name):
if type(cell) is _dill.CellType and name == 'cell_contents':
try:
cell.cell_contents
except:
return
def cell_deleter():
%s
del cell # pylint: disable=unused-variable
func = _dill.FunctionType(cell_deleter.__code__, globals(), "", None, (cell,)) # same as cell_deleter, but with cell being the cell's contents
func()
else:
delattr(cell, name)''' % __nonlocal)
move_to(_dill)(_delattr)

else:
# Likely PyPy 2.7. Simulate the nonlocal keyword with bytecode
# manipulation.

# The following function is based on 'cell_set' from 'cloudpickle'
# https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L393-L482
# Copyright (c) 2012, Regents of the University of California.
# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
@move_to(_dill)
def _setattr(cell, name, value):
if type(cell) is _dill.CellType and name == 'cell_contents':
_cell_set = _dill.FunctionType(
_cell_set_template_code, {}, '_cell_set', (), (cell,),)
_cell_set(value)
else:
setattr(cell, name, value)

def _cell_set_factory(value):
lambda: cell
cell = value

co = _cell_set_factory.__code__

_cell_set_template_code = _dill.CodeType(
co.co_argcount,
co.co_nlocals,
co.co_stacksize,
co.co_flags,
co.co_code,
co.co_consts,
co.co_names,
co.co_varnames,
co.co_filename,
co.co_name,
co.co_firstlineno,
co.co_lnotab,
co.co_cellvars, # co_freevars is initialized with co_cellvars
(), # co_cellvars is made empty
)

del co

@move_to(_dill)
def _delattr(cell, name):
if type(cell) is _dill.CellType and name == 'cell_contents':
pass
else:
delattr(cell, name)

_setattr = Getattr(_dill, '_setattr', setattr)
_delattr = Getattr(_dill, '_delattr', delattr)
Loading

0 comments on commit fc46578

Please sign in to comment.