Update build-tools to ab/10712949
Upgrades python to 3.11
https://ci.android.com/builds/branches/aosp-build-tools-release/grid?head=10712949&tail=10712949
Bug: 278602456
Test: treehugger
Change-Id: I65e52ef6c3975dc11ac2fe8beac9b52887a51214
diff --git a/Android.bp b/Android.bp
index 188a7c6..a6a0cfd 100644
--- a/Android.bp
+++ b/Android.bp
@@ -274,6 +274,6 @@
python_library_host {
name: "py3-stdlib-prebuilt",
is_internal: true,
- pkg_path: "stdlib",
+ pkg_path: "python3.11",
srcs: [":py3-stdlib-prebuilt-srcs"],
}
diff --git a/common/py3-stdlib/__future__.py b/common/py3-stdlib/__future__.py
index 97dc90c..39720a5 100644
--- a/common/py3-stdlib/__future__.py
+++ b/common/py3-stdlib/__future__.py
@@ -33,7 +33,7 @@
to use the feature in question, but may continue to use such imports.
MandatoryRelease may also be None, meaning that a planned feature got
-dropped.
+dropped or that the release version is undetermined.
Instances of class _Feature have two corresponding methods,
.getOptionalRelease() and .getMandatoryRelease().
@@ -96,7 +96,7 @@
"""Return release in which this feature will become mandatory.
This is a 5-tuple, of the same form as sys.version_info, or, if
- the feature was dropped, is None.
+ the feature was dropped, or the release date is undetermined, is None.
"""
return self.mandatory
@@ -143,5 +143,5 @@
CO_FUTURE_GENERATOR_STOP)
annotations = _Feature((3, 7, 0, "beta", 1),
- (3, 11, 0, "alpha", 0),
+ None,
CO_FUTURE_ANNOTATIONS)
diff --git a/common/py3-stdlib/__hello__.py b/common/py3-stdlib/__hello__.py
new file mode 100644
index 0000000..c09d6a4
--- /dev/null
+++ b/common/py3-stdlib/__hello__.py
@@ -0,0 +1,16 @@
+initialized = True
+
+class TestFrozenUtf8_1:
+ """\u00b6"""
+
+class TestFrozenUtf8_2:
+ """\u03c0"""
+
+class TestFrozenUtf8_4:
+ """\U0001f600"""
+
+def main():
+ print("Hello world!")
+
+if __name__ == '__main__':
+ main()
diff --git a/common/py3-stdlib/_aix_support.py b/common/py3-stdlib/_aix_support.py
index 4550493..1d8482f 100644
--- a/common/py3-stdlib/_aix_support.py
+++ b/common/py3-stdlib/_aix_support.py
@@ -15,8 +15,9 @@
# type: (List[int], int) -> str
# Infer the ABI bitwidth from maxsize (assuming 64 bit as the default)
_sz = 32 if sys.maxsize == (2**31-1) else 64
+ _bd = bd if bd != 0 else 9988
# vrtl[version, release, technology_level]
- return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], bd, _sz)
+ return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz)
# extract version, release and technology level from a VRMF string
@@ -26,32 +27,33 @@
return [int(v[-1]), int(r), int(tl)]
-def _aix_bosmp64():
+def _aix_bos_rte():
# type: () -> Tuple[str, int]
"""
Return a Tuple[str, int] e.g., ['7.1.4.34', 1806]
- The fileset bos.mp64 is the AIX kernel. It's VRMF and builddate
- reflect the current ABI levels of the runtime environment.
+ The fileset bos.rte represents the current AIX run-time level. It's VRMF and
+ builddate reflect the current ABI levels of the runtime environment.
+ If no builddate is found give a value that will satisfy pep425 related queries
"""
- # We expect all AIX systems to have lslpp installed in this location
- out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.mp64"])
+ # All AIX systems to have lslpp installed in this location
+ out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"])
out = out.decode("utf-8")
out = out.strip().split(":") # type: ignore
- # Use str() and int() to help mypy see types
- return (str(out[2]), int(out[-1]))
+ _bd = int(out[-1]) if out[-1] != '' else 9988
+ return (str(out[2]), _bd)
def aix_platform():
# type: () -> str
"""
AIX filesets are identified by four decimal values: V.R.M.F.
- V (version) and R (release) can be retreived using ``uname``
+ V (version) and R (release) can be retrieved using ``uname``
Since 2007, starting with AIX 5.3 TL7, the M value has been
- included with the fileset bos.mp64 and represents the Technology
+ included with the fileset bos.rte and represents the Technology
Level (TL) of AIX. The F (Fix) value also increases, but is not
relevant for comparing releases and binary compatibility.
For binary compatibility the so-called builddate is needed.
- Again, the builddate of an AIX release is associated with bos.mp64.
+ Again, the builddate of an AIX release is associated with bos.rte.
AIX ABI compatibility is described as guaranteed at: https://www.ibm.com/\
support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html
@@ -60,7 +62,7 @@
e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit
and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit
"""
- vrmf, bd = _aix_bosmp64()
+ vrmf, bd = _aix_bos_rte()
return _aix_tag(_aix_vrtl(vrmf), bd)
@@ -79,7 +81,7 @@
Return the platform_tag of the system Python was built on.
"""
# AIX_BUILDDATE is defined by configure with:
- # lslpp -Lcq bos.mp64 | awk -F: '{ print $NF }'
+ # lslpp -Lcq bos.rte | awk -F: '{ print $NF }'
build_date = sysconfig.get_config_var("AIX_BUILDDATE")
try:
build_date = int(build_date)
diff --git a/common/py3-stdlib/_collections_abc.py b/common/py3-stdlib/_collections_abc.py
index 72fd633..e96e4c3 100644
--- a/common/py3-stdlib/_collections_abc.py
+++ b/common/py3-stdlib/_collections_abc.py
@@ -430,27 +430,13 @@
raise TypeError(
"Callable must be used as Callable[[arg, ...], result].")
t_args, t_result = args
- if isinstance(t_args, list):
+ if isinstance(t_args, (tuple, list)):
args = (*t_args, t_result)
elif not _is_param_expr(t_args):
raise TypeError(f"Expected a list of types, an ellipsis, "
f"ParamSpec, or Concatenate. Got {t_args}")
return super().__new__(cls, origin, args)
- @property
- def __parameters__(self):
- params = []
- for arg in self.__args__:
- if isinstance(arg, type) and not isinstance(arg, GenericAlias):
- continue
- # Looks like a genericalias
- if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple):
- params.extend(arg.__parameters__)
- else:
- if _is_typevarlike(arg):
- params.append(arg)
- return tuple(dict.fromkeys(params))
-
def __repr__(self):
if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
return super().__repr__()
@@ -470,60 +456,24 @@
# code is copied from typing's _GenericAlias and the builtin
# types.GenericAlias.
- # A special case in PEP 612 where if X = Callable[P, int],
- # then X[int, str] == X[[int, str]].
- param_len = len(self.__parameters__)
- if param_len == 0:
- raise TypeError(f'{self} is not a generic class')
if not isinstance(item, tuple):
item = (item,)
- if (param_len == 1 and _is_param_expr(self.__parameters__[0])
+ # A special case in PEP 612 where if X = Callable[P, int],
+ # then X[int, str] == X[[int, str]].
+ if (len(self.__parameters__) == 1
+ and _is_param_expr(self.__parameters__[0])
and item and not _is_param_expr(item[0])):
- item = (list(item),)
- item_len = len(item)
- if item_len != param_len:
- raise TypeError(f'Too {"many" if item_len > param_len else "few"}'
- f' arguments for {self};'
- f' actual {item_len}, expected {param_len}')
- subst = dict(zip(self.__parameters__, item))
- new_args = []
- for arg in self.__args__:
- if isinstance(arg, type) and not isinstance(arg, GenericAlias):
- new_args.append(arg)
- continue
- if _is_typevarlike(arg):
- if _is_param_expr(arg):
- arg = subst[arg]
- if not _is_param_expr(arg):
- raise TypeError(f"Expected a list of types, an ellipsis, "
- f"ParamSpec, or Concatenate. Got {arg}")
- else:
- arg = subst[arg]
- # Looks like a GenericAlias
- elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple):
- subparams = arg.__parameters__
- if subparams:
- subargs = tuple(subst[x] for x in subparams)
- arg = arg[subargs]
- if isinstance(arg, tuple):
- new_args.extend(arg)
- else:
- new_args.append(arg)
+ item = (item,)
+
+ new_args = super().__getitem__(item).__args__
# args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
- if not isinstance(new_args[0], list):
+ if not isinstance(new_args[0], (tuple, list)):
t_result = new_args[-1]
t_args = new_args[:-1]
new_args = (t_args, t_result)
return _CallableGenericAlias(Callable, tuple(new_args))
-
-def _is_typevarlike(arg):
- obj = type(arg)
- # looks like a TypeVar/ParamSpec
- return (obj.__module__ == 'typing'
- and obj.__name__ in {'ParamSpec', 'TypeVar'})
-
def _is_param_expr(obj):
"""Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
``_ConcatenateGenericAlias`` from typing.py
@@ -1072,10 +1022,10 @@
while stop is None or i < stop:
try:
v = self[i]
- if v is value or v == value:
- return i
except IndexError:
break
+ if v is value or v == value:
+ return i
i += 1
raise ValueError
diff --git a/common/py3-stdlib/_compat_pickle.py b/common/py3-stdlib/_compat_pickle.py
index f68496a..65a94b6 100644
--- a/common/py3-stdlib/_compat_pickle.py
+++ b/common/py3-stdlib/_compat_pickle.py
@@ -249,3 +249,4 @@
for excname in PYTHON3_IMPORTERROR_EXCEPTIONS:
REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'ImportError')
+del excname
diff --git a/common/py3-stdlib/_pydecimal.py b/common/py3-stdlib/_pydecimal.py
index 3d6cece..f9d6c99 100644
--- a/common/py3-stdlib/_pydecimal.py
+++ b/common/py3-stdlib/_pydecimal.py
@@ -441,6 +441,10 @@
_current_context_var = contextvars.ContextVar('decimal_context')
+_context_attributes = frozenset(
+ ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps']
+)
+
def getcontext():
"""Returns this thread's context.
@@ -464,7 +468,7 @@
del contextvars # Don't contaminate the namespace
-def localcontext(ctx=None):
+def localcontext(ctx=None, **kwargs):
"""Return a context manager for a copy of the supplied context
Uses a copy of the current context if no context is specified
@@ -500,8 +504,14 @@
>>> print(getcontext().prec)
28
"""
- if ctx is None: ctx = getcontext()
- return _ContextManager(ctx)
+ if ctx is None:
+ ctx = getcontext()
+ ctx_manager = _ContextManager(ctx)
+ for key, value in kwargs.items():
+ if key not in _context_attributes:
+ raise TypeError(f"'{key}' is an invalid keyword argument for this function")
+ setattr(ctx_manager.new_context, key, value)
+ return ctx_manager
##### Decimal class #######################################################
@@ -2230,7 +2240,7 @@
if xe != 0 and len(str(abs(yc*xe))) <= -ye:
return None
xc_bits = _nbits(xc)
- if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:
+ if len(str(abs(yc)*xc_bits)) <= -ye:
return None
m, n = yc, 10**(-ye)
while m % 2 == n % 2 == 0:
@@ -2243,7 +2253,7 @@
# compute nth root of xc*10**xe
if n > 1:
# if 1 < xc < 2**n then xc isn't an nth power
- if xc != 1 and xc_bits <= n:
+ if xc_bits <= n:
return None
xe, rem = divmod(xe, n)
@@ -3795,6 +3805,10 @@
# represented in fixed point; rescale them to 0e0.
if not self and self._exp > 0 and spec['type'] in 'fF%':
self = self._rescale(0, rounding)
+ if not self and spec['no_neg_0'] and self._sign:
+ adjusted_sign = 0
+ else:
+ adjusted_sign = self._sign
# figure out placement of the decimal point
leftdigits = self._exp + len(self._int)
@@ -3825,7 +3839,7 @@
# done with the decimal-specific stuff; hand over the rest
# of the formatting to the _format_number function
- return _format_number(self._sign, intpart, fracpart, exp, spec)
+ return _format_number(adjusted_sign, intpart, fracpart, exp, spec)
def _dec_from_triple(sign, coefficient, exponent, special=False):
"""Create a decimal instance directly, without any validation,
@@ -6143,7 +6157,7 @@
#
# A format specifier for Decimal looks like:
#
-# [[fill]align][sign][#][0][minimumwidth][,][.precision][type]
+# [[fill]align][sign][z][#][0][minimumwidth][,][.precision][type]
_parse_format_specifier_regex = re.compile(r"""\A
(?:
@@ -6151,6 +6165,7 @@
(?P<align>[<>=^])
)?
(?P<sign>[-+ ])?
+(?P<no_neg_0>z)?
(?P<alt>\#)?
(?P<zeropad>0)?
(?P<minimumwidth>(?!0)\d+)?
diff --git a/common/py3-stdlib/_pyio.py b/common/py3-stdlib/_pyio.py
index fb867fb..0bfdeaa 100644
--- a/common/py3-stdlib/_pyio.py
+++ b/common/py3-stdlib/_pyio.py
@@ -44,8 +44,9 @@
"""
A helper function to choose the text encoding.
- When encoding is not None, just return it.
- Otherwise, return the default text encoding (i.e. "locale").
+ When encoding is not None, this function returns it.
+ Otherwise, this function returns the default text encoding
+ (i.e. "locale" or "utf-8" depends on UTF-8 mode).
This function emits an EncodingWarning if *encoding* is None and
sys.flags.warn_default_encoding is true.
@@ -55,7 +56,10 @@
However, please consider using encoding="utf-8" for new APIs.
"""
if encoding is None:
- encoding = "locale"
+ if sys.flags.utf8_mode:
+ encoding = "utf-8"
+ else:
+ encoding = "locale"
if sys.flags.warn_default_encoding:
import warnings
warnings.warn("'encoding' argument not specified.",
@@ -101,7 +105,6 @@
'b' binary mode
't' text mode (default)
'+' open a disk file for updating (reading and writing)
- 'U' universal newline mode (deprecated)
========= ===============================================================
The default mode is 'rt' (open for reading text). For binary random
@@ -117,10 +120,6 @@
returned as strings, the bytes having been first decoded using a
platform-dependent encoding or using the specified encoding if given.
- 'U' mode is deprecated and will raise an exception in future versions
- of Python. It has no effect in Python 3. Use newline to control
- universal newlines mode.
-
buffering is an optional integer used to set the buffering policy.
Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
line buffering (only usable in text mode), and an integer > 1 to indicate
@@ -206,7 +205,7 @@
if errors is not None and not isinstance(errors, str):
raise TypeError("invalid errors: %r" % errors)
modes = set(mode)
- if modes - set("axrwb+tU") or len(mode) > len(modes):
+ if modes - set("axrwb+t") or len(mode) > len(modes):
raise ValueError("invalid mode: %r" % mode)
creating = "x" in modes
reading = "r" in modes
@@ -215,13 +214,6 @@
updating = "+" in modes
text = "t" in modes
binary = "b" in modes
- if "U" in modes:
- if creating or writing or appending or updating:
- raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'")
- import warnings
- warnings.warn("'U' mode is deprecated",
- DeprecationWarning, 2)
- reading = True
if text and binary:
raise ValueError("can't have text and binary mode at once")
if creating + reading + writing + appending > 1:
@@ -324,7 +316,7 @@
global OpenWrapper
OpenWrapper = open
return OpenWrapper
- raise AttributeError(name)
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
# In normal operation, both `UnsupportedOperation`s should be bound to the
@@ -1996,7 +1988,7 @@
r"""Character and line based layer over a BufferedIOBase object, buffer.
encoding gives the name of the encoding that the stream will be
- decoded or encoded with. It defaults to locale.getpreferredencoding(False).
+ decoded or encoded with. It defaults to locale.getencoding().
errors determines the strictness of encoding and decoding (see the
codecs.register) and defaults to "strict".
@@ -2030,19 +2022,7 @@
encoding = text_encoding(encoding)
if encoding == "locale":
- try:
- encoding = os.device_encoding(buffer.fileno()) or "locale"
- except (AttributeError, UnsupportedOperation):
- pass
-
- if encoding == "locale":
- try:
- import locale
- except ImportError:
- # Importing locale may fail if Python is being built
- encoding = "utf-8"
- else:
- encoding = locale.getpreferredencoding(False)
+ encoding = self._get_locale_encoding()
if not isinstance(encoding, str):
raise ValueError("invalid encoding: %r" % encoding)
@@ -2175,6 +2155,8 @@
else:
if not isinstance(encoding, str):
raise TypeError("invalid encoding: %r" % encoding)
+ if encoding == "locale":
+ encoding = self._get_locale_encoding()
if newline is Ellipsis:
newline = self._readnl
@@ -2279,6 +2261,15 @@
self._decoded_chars_used += len(chars)
return chars
+ def _get_locale_encoding(self):
+ try:
+ import locale
+ except ImportError:
+ # Importing locale may fail if Python is being built
+ return "utf-8"
+ else:
+ return locale.getencoding()
+
def _rewind_decoded_chars(self, n):
"""Rewind the _decoded_chars buffer."""
if self._decoded_chars_used < n:
diff --git a/common/py3-stdlib/_weakrefset.py b/common/py3-stdlib/_weakrefset.py
index 2a27684..489eec7 100644
--- a/common/py3-stdlib/_weakrefset.py
+++ b/common/py3-stdlib/_weakrefset.py
@@ -80,8 +80,7 @@
return wr in self.data
def __reduce__(self):
- return (self.__class__, (list(self),),
- getattr(self, '__dict__', None))
+ return self.__class__, (list(self),), self.__getstate__()
def add(self, item):
if self._pending_removals:
diff --git a/common/py3-stdlib/abc.py b/common/py3-stdlib/abc.py
index 3c552ce..f8a4e11 100644
--- a/common/py3-stdlib/abc.py
+++ b/common/py3-stdlib/abc.py
@@ -18,7 +18,7 @@
class C(metaclass=ABCMeta):
@abstractmethod
- def my_abstract_method(self, ...):
+ def my_abstract_method(self, arg1, arg2, argN):
...
"""
funcobj.__isabstractmethod__ = True
@@ -102,7 +102,7 @@
implementations defined by the registering ABC be callable (not
even via super()).
"""
- def __new__(mcls, name, bases, namespace, **kwargs):
+ def __new__(mcls, name, bases, namespace, /, **kwargs):
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
_abc_init(cls)
return cls
diff --git a/common/py3-stdlib/aifc.py b/common/py3-stdlib/aifc.py
index ed5da7d..5254987 100644
--- a/common/py3-stdlib/aifc.py
+++ b/common/py3-stdlib/aifc.py
@@ -140,6 +140,10 @@
__all__ = ["Error", "open"]
+
+warnings._deprecated(__name__, remove=(3, 13))
+
+
class Error(Exception):
pass
@@ -251,7 +255,9 @@
_write_ulong(f, himant)
_write_ulong(f, lomant)
-from chunk import Chunk
+with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+ from chunk import Chunk
from collections import namedtuple
_aifc_params = namedtuple('_aifc_params',
@@ -447,21 +453,33 @@
#
def _alaw2lin(self, data):
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
return audioop.alaw2lin(data, 2)
def _ulaw2lin(self, data):
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
return audioop.ulaw2lin(data, 2)
def _adpcm2lin(self, data):
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
if not hasattr(self, '_adpcmstate'):
# first time
self._adpcmstate = None
data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
return data
+ def _sowt2lin(self, data):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
+ return audioop.byteswap(data, 2)
+
def _read_comm_chunk(self, chunk):
self._nchannels = _read_short(chunk)
self._nframes = _read_long(chunk)
@@ -497,6 +515,8 @@
self._convert = self._ulaw2lin
elif self._comptype in (b'alaw', b'ALAW'):
self._convert = self._alaw2lin
+ elif self._comptype in (b'sowt', b'SOWT'):
+ self._convert = self._sowt2lin
else:
raise Error('unsupported compression type')
self._sampwidth = 2
@@ -659,7 +679,7 @@
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if comptype not in (b'NONE', b'ulaw', b'ULAW',
- b'alaw', b'ALAW', b'G722'):
+ b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
raise Error('unsupported compression type')
self._comptype = comptype
self._compname = compname
@@ -680,7 +700,7 @@
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
if comptype not in (b'NONE', b'ulaw', b'ULAW',
- b'alaw', b'ALAW', b'G722'):
+ b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
raise Error('unsupported compression type')
self.setnchannels(nchannels)
self.setsampwidth(sampwidth)
@@ -764,28 +784,43 @@
#
def _lin2alaw(self, data):
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
return audioop.lin2alaw(data, 2)
def _lin2ulaw(self, data):
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
return audioop.lin2ulaw(data, 2)
def _lin2adpcm(self, data):
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
if not hasattr(self, '_adpcmstate'):
self._adpcmstate = None
data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
return data
+ def _lin2sowt(self, data):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
+ return audioop.byteswap(data, 2)
+
def _ensure_header_written(self, datasize):
if not self._nframeswritten:
- if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
+ if self._comptype in (b'ULAW', b'ulaw',
+ b'ALAW', b'alaw', b'G722',
+ b'sowt', b'SOWT'):
if not self._sampwidth:
self._sampwidth = 2
if self._sampwidth != 2:
raise Error('sample width must be 2 when compressing '
- 'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
+ 'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
+ 'or G7.22 (ADPCM)')
if not self._nchannels:
raise Error('# channels not specified')
if not self._sampwidth:
@@ -801,6 +836,8 @@
self._convert = self._lin2ulaw
elif self._comptype in (b'alaw', b'ALAW'):
self._convert = self._lin2alaw
+ elif self._comptype in (b'sowt', b'SOWT'):
+ self._convert = self._lin2sowt
def _write_header(self, initlength):
if self._aifc and self._comptype != b'NONE':
diff --git a/common/py3-stdlib/argparse.py b/common/py3-stdlib/argparse.py
index 736bf5a..9962e61 100644
--- a/common/py3-stdlib/argparse.py
+++ b/common/py3-stdlib/argparse.py
@@ -89,6 +89,8 @@
import re as _re
import sys as _sys
+import warnings
+
from gettext import gettext as _, ngettext
SUPPRESS = '==SUPPRESS=='
@@ -151,6 +153,7 @@
# Formatting Help
# ===============
+
class HelpFormatter(object):
"""Formatter for generating usage messages and argument help strings.
@@ -700,8 +703,19 @@
"""
def _get_help_string(self, action):
+ """
+ Add the default value to the option help message.
+
+ ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't
+ already present. This code will do that, detecting cornercases to
+ prevent duplicates or cases where it wouldn't make sense to the end
+ user.
+ """
help = action.help
- if '%(default)' not in action.help:
+ if help is None:
+ help = ''
+
+ if '%(default)' not in help:
if action.default is not SUPPRESS:
defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
if action.option_strings or action.nargs in defaulting_nargs:
@@ -709,6 +723,7 @@
return help
+
class MetavarTypeHelpFormatter(HelpFormatter):
"""Help message formatter which uses the argument 'type' as the default
metavar value (instead of the argument 'dest')
@@ -724,7 +739,6 @@
return action.type.__name__
-
# =====================
# Options and Arguments
# =====================
@@ -759,7 +773,7 @@
if self.argument_name is None:
format = '%(message)s'
else:
- format = 'argument %(argument_name)s: %(message)s'
+ format = _('argument %(argument_name)s: %(message)s')
return format % dict(message=self.message,
argument_name=self.argument_name)
@@ -867,6 +881,7 @@
def __call__(self, parser, namespace, values, option_string=None):
raise NotImplementedError(_('.__call__() not defined'))
+
class BooleanOptionalAction(Action):
def __init__(self,
option_strings,
@@ -886,9 +901,6 @@
option_string = '--no-' + option_string[2:]
_option_strings.append(option_string)
- if help is not None and default is not None and default is not SUPPRESS:
- help += " (default: %(default)s)"
-
super().__init__(
option_strings=_option_strings,
dest=dest,
@@ -900,6 +912,7 @@
help=help,
metavar=metavar)
+
def __call__(self, parser, namespace, values, option_string=None):
if option_string in self.option_strings:
setattr(namespace, self.dest, not option_string.startswith('--no-'))
@@ -948,7 +961,7 @@
def __init__(self,
option_strings,
dest,
- const,
+ const=None,
default=None,
required=False,
help=None,
@@ -1043,7 +1056,7 @@
def __init__(self,
option_strings,
dest,
- const,
+ const=None,
default=None,
required=False,
help=None,
@@ -1175,6 +1188,13 @@
aliases = kwargs.pop('aliases', ())
+ if name in self._name_parser_map:
+ raise ArgumentError(self, _('conflicting subparser: %s') % name)
+ for alias in aliases:
+ if alias in self._name_parser_map:
+ raise ArgumentError(
+ self, _('conflicting subparser alias: %s') % alias)
+
# create a pseudo-action to hold the choice help
if 'help' in kwargs:
help = kwargs.pop('help')
@@ -1655,6 +1675,14 @@
super(_ArgumentGroup, self)._remove_action(action)
self._group_actions.remove(action)
+ def add_argument_group(self, *args, **kwargs):
+ warnings.warn(
+ "Nesting argument groups is deprecated.",
+ category=DeprecationWarning,
+ stacklevel=2
+ )
+ return super().add_argument_group(*args, **kwargs)
+
class _MutuallyExclusiveGroup(_ArgumentGroup):
@@ -1675,6 +1703,14 @@
self._container._remove_action(action)
self._group_actions.remove(action)
+ def add_mutually_exclusive_group(self, *args, **kwargs):
+ warnings.warn(
+ "Nesting mutually exclusive groups is deprecated.",
+ category=DeprecationWarning,
+ stacklevel=2
+ )
+ return super().add_mutually_exclusive_group(*args, **kwargs)
+
class ArgumentParser(_AttributeHolder, _ActionsContainer):
"""Object for parsing command line strings into Python objects.
@@ -1864,8 +1900,7 @@
if self.exit_on_error:
try:
namespace, args = self._parse_known_args(args, namespace)
- except ArgumentError:
- err = _sys.exc_info()[1]
+ except ArgumentError as err:
self.error(str(err))
else:
namespace, args = self._parse_known_args(args, namespace)
@@ -2144,8 +2179,7 @@
arg_strings.append(arg)
arg_strings = self._read_args_from_files(arg_strings)
new_arg_strings.extend(arg_strings)
- except OSError:
- err = _sys.exc_info()[1]
+ except OSError as err:
self.error(str(err))
# return the modified argument list
@@ -2495,9 +2529,9 @@
result = type_func(arg_string)
# ArgumentTypeErrors indicate errors
- except ArgumentTypeError:
+ except ArgumentTypeError as err:
name = getattr(action.type, '__name__', repr(action.type))
- msg = str(_sys.exc_info()[1])
+ msg = str(err)
raise ArgumentError(action, msg)
# TypeErrors or ValueErrors also indicate errors
@@ -2568,9 +2602,11 @@
def _print_message(self, message, file=None):
if message:
- if file is None:
- file = _sys.stderr
- file.write(message)
+ file = file or _sys.stderr
+ try:
+ file.write(message)
+ except (AttributeError, OSError):
+ pass
# ===============
# Exiting methods
diff --git a/common/py3-stdlib/ast.py b/common/py3-stdlib/ast.py
index 4f5f982..623b9a1 100644
--- a/common/py3-stdlib/ast.py
+++ b/common/py3-stdlib/ast.py
@@ -27,7 +27,7 @@
import sys
from _ast import *
from contextlib import contextmanager, nullcontext
-from enum import IntEnum, auto
+from enum import IntEnum, auto, _simple_enum
def parse(source, filename='<unknown>', mode='exec', *,
@@ -644,10 +644,12 @@
# We unparse those infinities to INFSTR.
_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
-class _Precedence(IntEnum):
+@_simple_enum(IntEnum)
+class _Precedence:
"""Precedence table that originated from python grammar."""
- TUPLE = auto()
+ NAMED_EXPR = auto() # <target> := <expr1>
+ TUPLE = auto() # <expr1>, <expr2>
YIELD = auto() # 'yield', 'yield from'
TEST = auto() # 'if'-'else', 'lambda'
OR = auto() # 'or'
@@ -685,11 +687,11 @@
def __init__(self, *, _avoid_backslashes=False):
self._source = []
- self._buffer = []
self._precedences = {}
self._type_ignores = {}
self._indent = 0
self._avoid_backslashes = _avoid_backslashes
+ self._in_try_star = False
def interleave(self, inter, f, seq):
"""Call f on each item in seq, calling inter() in between."""
@@ -724,18 +726,19 @@
self.maybe_newline()
self.write(" " * self._indent + text)
- def write(self, text):
- """Append a piece of text"""
- self._source.append(text)
+ def write(self, *text):
+ """Add new source parts"""
+ self._source.extend(text)
- def buffer_writer(self, text):
- self._buffer.append(text)
+ @contextmanager
+ def buffered(self, buffer = None):
+ if buffer is None:
+ buffer = []
- @property
- def buffer(self):
- value = "".join(self._buffer)
- self._buffer.clear()
- return value
+ original_source = self._source
+ self._source = buffer
+ yield buffer
+ self._source = original_source
@contextmanager
def block(self, *, extra = None):
@@ -845,7 +848,7 @@
self.traverse(node.value)
def visit_NamedExpr(self, node):
- with self.require_parens(_Precedence.TUPLE, node):
+ with self.require_parens(_Precedence.NAMED_EXPR, node):
self.set_precedence(_Precedence.ATOM, node.target, node.value)
self.traverse(node.target)
self.write(" := ")
@@ -866,6 +869,7 @@
def visit_Assign(self, node):
self.fill()
for target in node.targets:
+ self.set_precedence(_Precedence.TUPLE, target)
self.traverse(target)
self.write(" = ")
self.traverse(node.value)
@@ -958,7 +962,7 @@
self.write(" from ")
self.traverse(node.cause)
- def visit_Try(self, node):
+ def do_visit_try(self, node):
self.fill("try")
with self.block():
self.traverse(node.body)
@@ -973,8 +977,24 @@
with self.block():
self.traverse(node.finalbody)
+ def visit_Try(self, node):
+ prev_in_try_star = self._in_try_star
+ try:
+ self._in_try_star = False
+ self.do_visit_try(node)
+ finally:
+ self._in_try_star = prev_in_try_star
+
+ def visit_TryStar(self, node):
+ prev_in_try_star = self._in_try_star
+ try:
+ self._in_try_star = True
+ self.do_visit_try(node)
+ finally:
+ self._in_try_star = prev_in_try_star
+
def visit_ExceptHandler(self, node):
- self.fill("except")
+ self.fill("except*" if self._in_try_star else "except")
if node.type:
self.write(" ")
self.traverse(node.type)
@@ -1037,6 +1057,7 @@
def _for_helper(self, fill, node):
self.fill(fill)
+ self.set_precedence(_Precedence.TUPLE, node.target)
self.traverse(node.target)
self.write(" in ")
self.traverse(node.iter)
@@ -1134,9 +1155,9 @@
def visit_JoinedStr(self, node):
self.write("f")
if self._avoid_backslashes:
- self._fstring_JoinedStr(node, self.buffer_writer)
- self._write_str_avoiding_backslashes(self.buffer)
- return
+ with self.buffered() as buffer:
+ self._write_fstring_inner(node)
+ return self._write_str_avoiding_backslashes("".join(buffer))
# If we don't need to avoid backslashes globally (i.e., we only need
# to avoid them inside FormattedValues), it's cosmetically preferred
@@ -1144,60 +1165,62 @@
# for cases like: f"{x}\n". To accomplish this, we keep track of what
# in our buffer corresponds to FormattedValues and what corresponds to
# Constant parts of the f-string, and allow escapes accordingly.
- buffer = []
+ fstring_parts = []
for value in node.values:
- meth = getattr(self, "_fstring_" + type(value).__name__)
- meth(value, self.buffer_writer)
- buffer.append((self.buffer, isinstance(value, Constant)))
- new_buffer = []
- quote_types = _ALL_QUOTES
- for value, is_constant in buffer:
- # Repeatedly narrow down the list of possible quote_types
- value, quote_types = self._str_literal_helper(
- value, quote_types=quote_types,
- escape_special_whitespace=is_constant
+ with self.buffered() as buffer:
+ self._write_fstring_inner(value)
+ fstring_parts.append(
+ ("".join(buffer), isinstance(value, Constant))
)
- new_buffer.append(value)
- value = "".join(new_buffer)
+
+ new_fstring_parts = []
+ quote_types = list(_ALL_QUOTES)
+ for value, is_constant in fstring_parts:
+ value, quote_types = self._str_literal_helper(
+ value,
+ quote_types=quote_types,
+ escape_special_whitespace=is_constant,
+ )
+ new_fstring_parts.append(value)
+
+ value = "".join(new_fstring_parts)
quote_type = quote_types[0]
self.write(f"{quote_type}{value}{quote_type}")
+ def _write_fstring_inner(self, node):
+ if isinstance(node, JoinedStr):
+ # for both the f-string itself, and format_spec
+ for value in node.values:
+ self._write_fstring_inner(value)
+ elif isinstance(node, Constant) and isinstance(node.value, str):
+ value = node.value.replace("{", "{{").replace("}", "}}")
+ self.write(value)
+ elif isinstance(node, FormattedValue):
+ self.visit_FormattedValue(node)
+ else:
+ raise ValueError(f"Unexpected node inside JoinedStr, {node!r}")
+
def visit_FormattedValue(self, node):
- self.write("f")
- self._fstring_FormattedValue(node, self.buffer_writer)
- self._write_str_avoiding_backslashes(self.buffer)
+ def unparse_inner(inner):
+ unparser = type(self)(_avoid_backslashes=True)
+ unparser.set_precedence(_Precedence.TEST.next(), inner)
+ return unparser.visit(inner)
- def _fstring_JoinedStr(self, node, write):
- for value in node.values:
- meth = getattr(self, "_fstring_" + type(value).__name__)
- meth(value, write)
-
- def _fstring_Constant(self, node, write):
- if not isinstance(node.value, str):
- raise ValueError("Constants inside JoinedStr should be a string.")
- value = node.value.replace("{", "{{").replace("}", "}}")
- write(value)
-
- def _fstring_FormattedValue(self, node, write):
- write("{")
- unparser = type(self)(_avoid_backslashes=True)
- unparser.set_precedence(_Precedence.TEST.next(), node.value)
- expr = unparser.visit(node.value)
- if expr.startswith("{"):
- write(" ") # Separate pair of opening brackets as "{ {"
- if "\\" in expr:
- raise ValueError("Unable to avoid backslash in f-string expression part")
- write(expr)
- if node.conversion != -1:
- conversion = chr(node.conversion)
- if conversion not in "sra":
- raise ValueError("Unknown f-string conversion.")
- write(f"!{conversion}")
- if node.format_spec:
- write(":")
- meth = getattr(self, "_fstring_" + type(node.format_spec).__name__)
- meth(node.format_spec, write)
- write("}")
+ with self.delimit("{", "}"):
+ expr = unparse_inner(node.value)
+ if "\\" in expr:
+ raise ValueError(
+ "Unable to avoid backslash in f-string expression part"
+ )
+ if expr.startswith("{"):
+ # Separate pair of opening brackets as "{ {"
+ self.write(" ")
+ self.write(expr)
+ if node.conversion != -1:
+ self.write(f"!{chr(node.conversion)}")
+ if node.format_spec:
+ self.write(":")
+ self._write_fstring_inner(node.format_spec)
def visit_Name(self, node):
self.write(node.id)
@@ -1320,7 +1343,11 @@
)
def visit_Tuple(self, node):
- with self.delimit("(", ")"):
+ with self.delimit_if(
+ "(",
+ ")",
+ len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE
+ ):
self.items_view(self.traverse, node.elts)
unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"}
@@ -1336,7 +1363,7 @@
operator_precedence = self.unop_precedence[operator]
with self.require_parens(operator_precedence, node):
self.write(operator)
- # factor prefixes (+, -, ~) shouldn't be seperated
+ # factor prefixes (+, -, ~) shouldn't be separated
# from the value they belong, (e.g: +1 instead of + 1)
if operator_precedence is not _Precedence.FACTOR:
self.write(" ")
@@ -1461,20 +1488,17 @@
self.traverse(e)
def visit_Subscript(self, node):
- def is_simple_tuple(slice_value):
- # when unparsing a non-empty tuple, the parentheses can be safely
- # omitted if there aren't any elements that explicitly requires
- # parentheses (such as starred expressions).
+ def is_non_empty_tuple(slice_value):
return (
isinstance(slice_value, Tuple)
and slice_value.elts
- and not any(isinstance(elt, Starred) for elt in slice_value.elts)
)
self.set_precedence(_Precedence.ATOM, node.value)
self.traverse(node.value)
with self.delimit("[", "]"):
- if is_simple_tuple(node.slice):
+ if is_non_empty_tuple(node.slice):
+ # parentheses can be omitted if the tuple isn't empty
self.items_view(self.traverse, node.slice.elts)
else:
self.traverse(node.slice)
@@ -1571,8 +1595,11 @@
def visit_Lambda(self, node):
with self.require_parens(_Precedence.TEST, node):
- self.write("lambda ")
- self.traverse(node.args)
+ self.write("lambda")
+ with self.buffered() as buffer:
+ self.traverse(node.args)
+ if buffer:
+ self.write(" ", *buffer)
self.write(": ")
self.set_precedence(_Precedence.TEST, node.body)
self.traverse(node.body)
diff --git a/common/py3-stdlib/asynchat.py b/common/py3-stdlib/asynchat.py
index e081e67..bed797e 100644
--- a/common/py3-stdlib/asynchat.py
+++ b/common/py3-stdlib/asynchat.py
@@ -48,12 +48,11 @@
import asyncore
from collections import deque
-from warnings import warn
-warn(
- 'The asynchat module is deprecated and will be removed in Python 3.12. '
- 'The recommended replacement is asyncio',
- DeprecationWarning,
- stacklevel=2)
+from warnings import _deprecated
+
+_DEPRECATION_MSG = ('The {name} module is deprecated and will be removed in '
+ 'Python {remove}. The recommended replacement is asyncio')
+_deprecated(__name__, _DEPRECATION_MSG, remove=(3, 12))
diff --git a/common/py3-stdlib/asyncio/__init__.py b/common/py3-stdlib/asyncio/__init__.py
index 200b14c..fed16ec 100644
--- a/common/py3-stdlib/asyncio/__init__.py
+++ b/common/py3-stdlib/asyncio/__init__.py
@@ -17,6 +17,8 @@
from .streams import *
from .subprocess import *
from .tasks import *
+from .taskgroups import *
+from .timeouts import *
from .threads import *
from .transports import *
@@ -33,6 +35,7 @@
subprocess.__all__ +
tasks.__all__ +
threads.__all__ +
+ timeouts.__all__ +
transports.__all__)
if sys.platform == 'win32': # pragma: no cover
diff --git a/common/py3-stdlib/asyncio/base_events.py b/common/py3-stdlib/asyncio/base_events.py
index 0890e9e..c5c7bd4 100644
--- a/common/py3-stdlib/asyncio/base_events.py
+++ b/common/py3-stdlib/asyncio/base_events.py
@@ -66,10 +66,6 @@
# Maximum timeout passed to select to avoid OS limitations
MAXIMUM_SELECT_TIMEOUT = 24 * 3600
-# Used for deprecation and removal of `loop.create_datagram_endpoint()`'s
-# *reuse_address* parameter
-_unset = object()
-
def _format_handle(handle):
cb = handle._callback
@@ -278,7 +274,7 @@
class Server(events.AbstractServer):
def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog,
- ssl_handshake_timeout):
+ ssl_handshake_timeout, ssl_shutdown_timeout=None):
self._loop = loop
self._sockets = sockets
self._active_count = 0
@@ -287,6 +283,7 @@
self._backlog = backlog
self._ssl_context = ssl_context
self._ssl_handshake_timeout = ssl_handshake_timeout
+ self._ssl_shutdown_timeout = ssl_shutdown_timeout
self._serving = False
self._serving_forever_fut = None
@@ -318,7 +315,8 @@
sock.listen(self._backlog)
self._loop._start_serving(
self._protocol_factory, sock, self._ssl_context,
- self, self._backlog, self._ssl_handshake_timeout)
+ self, self._backlog, self._ssl_handshake_timeout,
+ self._ssl_shutdown_timeout)
def get_loop(self):
return self._loop
@@ -428,18 +426,23 @@
"""Create a Future object attached to the loop."""
return futures.Future(loop=self)
- def create_task(self, coro, *, name=None):
+ def create_task(self, coro, *, name=None, context=None):
"""Schedule a coroutine object.
Return a task object.
"""
self._check_closed()
if self._task_factory is None:
- task = tasks.Task(coro, loop=self, name=name)
+ task = tasks.Task(coro, loop=self, name=name, context=context)
if task._source_traceback:
del task._source_traceback[-1]
else:
- task = self._task_factory(self, coro)
+ if context is None:
+ # Use legacy API if context is not needed
+ task = self._task_factory(self, coro)
+ else:
+ task = self._task_factory(self, coro, context=context)
+
tasks._set_task_name(task, name)
return task
@@ -472,6 +475,7 @@
*, server_side=False, server_hostname=None,
extra=None, server=None,
ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None,
call_connection_made=True):
"""Create SSL transport."""
raise NotImplementedError
@@ -718,6 +722,8 @@
Any positional arguments after the callback will be passed to
the callback when it is called.
"""
+ if delay is None:
+ raise TypeError('delay must not be None')
timer = self.call_at(self.time() + delay, callback, *args,
context=context)
if timer._source_traceback:
@@ -729,6 +735,8 @@
Absolute time corresponds to the event loop's time() method.
"""
+ if when is None:
+ raise TypeError("when cannot be None")
self._check_closed()
if self._debug:
self._check_thread()
@@ -822,11 +830,7 @@
def set_default_executor(self, executor):
if not isinstance(executor, concurrent.futures.ThreadPoolExecutor):
- warnings.warn(
- 'Using the default executor that is not an instance of '
- 'ThreadPoolExecutor is deprecated and will be prohibited '
- 'in Python 3.9',
- DeprecationWarning, 2)
+ raise TypeError('executor must be ThreadPoolExecutor instance')
self._default_executor = executor
def _getaddrinfo_debug(self, host, port, family, type, proto, flags):
@@ -986,6 +990,7 @@
proto=0, flags=0, sock=None,
local_addr=None, server_hostname=None,
ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None,
happy_eyeballs_delay=None, interleave=None):
"""Connect to a TCP server.
@@ -1021,6 +1026,10 @@
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if ssl_shutdown_timeout is not None and not ssl:
+ raise ValueError(
+ 'ssl_shutdown_timeout is only meaningful with ssl')
+
if sock is not None:
_check_ssl_socket(sock)
@@ -1102,7 +1111,8 @@
transport, protocol = await self._create_connection_transport(
sock, protocol_factory, ssl, server_hostname,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout)
if self._debug:
# Get the socket from the transport because SSL transport closes
# the old socket and creates a new SSL socket
@@ -1114,7 +1124,8 @@
async def _create_connection_transport(
self, sock, protocol_factory, ssl,
server_hostname, server_side=False,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
sock.setblocking(False)
@@ -1125,7 +1136,8 @@
transport = self._make_ssl_transport(
sock, protocol, sslcontext, waiter,
server_side=server_side, server_hostname=server_hostname,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout)
else:
transport = self._make_socket_transport(sock, protocol, waiter)
@@ -1216,7 +1228,8 @@
async def start_tls(self, transport, protocol, sslcontext, *,
server_side=False,
server_hostname=None,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
"""Upgrade transport to TLS.
Return a new transport that *protocol* should start using
@@ -1239,6 +1252,7 @@
self, protocol, sslcontext, waiter,
server_side, server_hostname,
ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout,
call_connection_made=False)
# Pause early so that "ssl_protocol.data_received()" doesn't
@@ -1262,7 +1276,7 @@
async def create_datagram_endpoint(self, protocol_factory,
local_addr=None, remote_addr=None, *,
family=0, proto=0, flags=0,
- reuse_address=_unset, reuse_port=None,
+ reuse_port=None,
allow_broadcast=None, sock=None):
"""Create datagram connection."""
if sock is not None:
@@ -1275,7 +1289,7 @@
# show the problematic kwargs in exception msg
opts = dict(local_addr=local_addr, remote_addr=remote_addr,
family=family, proto=proto, flags=flags,
- reuse_address=reuse_address, reuse_port=reuse_port,
+ reuse_port=reuse_port,
allow_broadcast=allow_broadcast)
problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v)
raise ValueError(
@@ -1312,8 +1326,8 @@
addr_infos = {} # Using order preserving dict
for idx, addr in ((0, local_addr), (1, remote_addr)):
if addr is not None:
- assert isinstance(addr, tuple) and len(addr) == 2, (
- '2-tuple is expected')
+ if not (isinstance(addr, tuple) and len(addr) == 2):
+ raise TypeError('2-tuple is expected')
infos = await self._ensure_resolved(
addr, family=family, type=socket.SOCK_DGRAM,
@@ -1338,19 +1352,6 @@
exceptions = []
- # bpo-37228
- if reuse_address is not _unset:
- if reuse_address:
- raise ValueError("Passing `reuse_address=True` is no "
- "longer supported, as the usage of "
- "SO_REUSEPORT in UDP poses a significant "
- "security concern.")
- else:
- warnings.warn("The *reuse_address* parameter has been "
- "deprecated as of 3.5.10 and is scheduled "
- "for removal in 3.11.", DeprecationWarning,
- stacklevel=2)
-
for ((family, proto),
(local_address, remote_address)) in addr_pairs_info:
sock = None
@@ -1437,6 +1438,7 @@
reuse_address=None,
reuse_port=None,
ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None,
start_serving=True):
"""Create a TCP server.
@@ -1460,6 +1462,10 @@
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if ssl_shutdown_timeout is not None and ssl is None:
+ raise ValueError(
+ 'ssl_shutdown_timeout is only meaningful with ssl')
+
if sock is not None:
_check_ssl_socket(sock)
@@ -1469,7 +1475,7 @@
'host/port and sock can not be specified at the same time')
if reuse_address is None:
- reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
+ reuse_address = os.name == "posix" and sys.platform != "cygwin"
sockets = []
if host == '':
hosts = [None]
@@ -1535,7 +1541,8 @@
sock.setblocking(False)
server = Server(self, sockets, protocol_factory,
- ssl, backlog, ssl_handshake_timeout)
+ ssl, backlog, ssl_handshake_timeout,
+ ssl_shutdown_timeout)
if start_serving:
server._start_serving()
# Skip one loop iteration so that all 'loop.add_reader'
@@ -1549,7 +1556,8 @@
async def connect_accepted_socket(
self, protocol_factory, sock,
*, ssl=None,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
if sock.type != socket.SOCK_STREAM:
raise ValueError(f'A Stream Socket was expected, got {sock!r}')
@@ -1557,12 +1565,17 @@
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if ssl_shutdown_timeout is not None and not ssl:
+ raise ValueError(
+ 'ssl_shutdown_timeout is only meaningful with ssl')
+
if sock is not None:
_check_ssl_socket(sock)
transport, protocol = await self._create_connection_transport(
sock, protocol_factory, ssl, '', server_side=True,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout)
if self._debug:
# Get the socket from the transport because SSL transport closes
# the old socket and creates a new SSL socket
diff --git a/common/py3-stdlib/asyncio/base_futures.py b/common/py3-stdlib/asyncio/base_futures.py
index 2c01ac9..cd811a7 100644
--- a/common/py3-stdlib/asyncio/base_futures.py
+++ b/common/py3-stdlib/asyncio/base_futures.py
@@ -42,16 +42,6 @@
return f'cb=[{cb}]'
-# bpo-42183: _repr_running is needed for repr protection
-# when a Future or Task result contains itself directly or indirectly.
-# The logic is borrowed from @reprlib.recursive_repr decorator.
-# Unfortunately, the direct decorator usage is impossible because of
-# AttributeError: '_asyncio.Task' object has no attribute '__module__' error.
-#
-# After fixing this thing we can return to the decorator based approach.
-_repr_running = set()
-
-
def _future_repr_info(future):
# (Future) -> str
"""helper function for Future.__repr__"""
@@ -60,17 +50,9 @@
if future._exception is not None:
info.append(f'exception={future._exception!r}')
else:
- key = id(future), get_ident()
- if key in _repr_running:
- result = '...'
- else:
- _repr_running.add(key)
- try:
- # use reprlib to limit the length of the output, especially
- # for very long strings
- result = reprlib.repr(future._result)
- finally:
- _repr_running.discard(key)
+ # use reprlib to limit the length of the output, especially
+ # for very long strings
+ result = reprlib.repr(future._result)
info.append(f'result={result}')
if future._callbacks:
info.append(_format_callbacks(future._callbacks))
@@ -78,3 +60,9 @@
frame = future._source_traceback[-1]
info.append(f'created at {frame[0]}:{frame[1]}')
return info
+
+
+@reprlib.recursive_repr()
+def _future_repr(future):
+ info = ' '.join(_future_repr_info(future))
+ return f'<{future.__class__.__name__} {info}>'
diff --git a/common/py3-stdlib/asyncio/base_subprocess.py b/common/py3-stdlib/asyncio/base_subprocess.py
index 14d5051..4c9b0dd 100644
--- a/common/py3-stdlib/asyncio/base_subprocess.py
+++ b/common/py3-stdlib/asyncio/base_subprocess.py
@@ -215,13 +215,8 @@
# object. On Python 3.6, it is required to avoid a ResourceWarning.
self._proc.returncode = returncode
self._call(self._protocol.process_exited)
- self._try_finish()
- # wake up futures waiting for wait()
- for waiter in self._exit_waiters:
- if not waiter.cancelled():
- waiter.set_result(returncode)
- self._exit_waiters = None
+ self._try_finish()
async def _wait(self):
"""Wait until the process exit and return the process return code.
@@ -247,6 +242,11 @@
try:
self._protocol.connection_lost(exc)
finally:
+ # wake up futures waiting for wait()
+ for waiter in self._exit_waiters:
+ if not waiter.cancelled():
+ waiter.set_result(self._returncode)
+ self._exit_waiters = None
self._loop = None
self._proc = None
self._protocol = None
diff --git a/common/py3-stdlib/asyncio/base_tasks.py b/common/py3-stdlib/asyncio/base_tasks.py
index 09bb171..26298e6 100644
--- a/common/py3-stdlib/asyncio/base_tasks.py
+++ b/common/py3-stdlib/asyncio/base_tasks.py
@@ -1,4 +1,5 @@
import linecache
+import reprlib
import traceback
from . import base_futures
@@ -8,7 +9,7 @@
def _task_repr_info(task):
info = base_futures._future_repr_info(task)
- if task._must_cancel:
+ if task.cancelling() and not task.done():
# replace status
info[0] = 'cancelling'
@@ -22,6 +23,12 @@
return info
+@reprlib.recursive_repr()
+def _task_repr(task):
+ info = ' '.join(_task_repr_info(task))
+ return f'<{task.__class__.__name__} {info}>'
+
+
def _task_get_stack(task, limit):
frames = []
if hasattr(task._coro, 'cr_frame'):
diff --git a/common/py3-stdlib/asyncio/constants.py b/common/py3-stdlib/asyncio/constants.py
index 33feed6..f171ead 100644
--- a/common/py3-stdlib/asyncio/constants.py
+++ b/common/py3-stdlib/asyncio/constants.py
@@ -15,10 +15,17 @@
# The default timeout matches that of Nginx.
SSL_HANDSHAKE_TIMEOUT = 60.0
+# Number of seconds to wait for SSL shutdown to complete
+# The default timeout mimics lingering_time
+SSL_SHUTDOWN_TIMEOUT = 30.0
+
# Used in sendfile fallback code. We use fallback for platforms
# that don't support sendfile, or for TLS connections.
SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256
+FLOW_CONTROL_HIGH_WATER_SSL_READ = 256 # KiB
+FLOW_CONTROL_HIGH_WATER_SSL_WRITE = 512 # KiB
+
# The enum should be here to break circular dependencies between
# base_events and sslproto
class _SendfileMode(enum.Enum):
diff --git a/common/py3-stdlib/asyncio/coroutines.py b/common/py3-stdlib/asyncio/coroutines.py
index 9664ea7..0e4b489 100644
--- a/common/py3-stdlib/asyncio/coroutines.py
+++ b/common/py3-stdlib/asyncio/coroutines.py
@@ -1,162 +1,19 @@
-__all__ = 'coroutine', 'iscoroutinefunction', 'iscoroutine'
+__all__ = 'iscoroutinefunction', 'iscoroutine'
import collections.abc
-import functools
import inspect
import os
import sys
import traceback
import types
-import warnings
-
-from . import base_futures
-from . import constants
-from . import format_helpers
-from .log import logger
def _is_debug_mode():
- # If you set _DEBUG to true, @coroutine will wrap the resulting
- # generator objects in a CoroWrapper instance (defined below). That
- # instance will log a message when the generator is never iterated
- # over, which may happen when you forget to use "await" or "yield from"
- # with a coroutine call.
- # Note that the value of the _DEBUG flag is taken
- # when the decorator is used, so to be of any use it must be set
- # before you define your coroutines. A downside of using this feature
- # is that tracebacks show entries for the CoroWrapper.__next__ method
- # when _DEBUG is true.
+ # See: https://docs.python.org/3/library/asyncio-dev.html#asyncio-debug-mode.
return sys.flags.dev_mode or (not sys.flags.ignore_environment and
bool(os.environ.get('PYTHONASYNCIODEBUG')))
-_DEBUG = _is_debug_mode()
-
-
-class CoroWrapper:
- # Wrapper for coroutine object in _DEBUG mode.
-
- def __init__(self, gen, func=None):
- assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen
- self.gen = gen
- self.func = func # Used to unwrap @coroutine decorator
- self._source_traceback = format_helpers.extract_stack(sys._getframe(1))
- self.__name__ = getattr(gen, '__name__', None)
- self.__qualname__ = getattr(gen, '__qualname__', None)
-
- def __repr__(self):
- coro_repr = _format_coroutine(self)
- if self._source_traceback:
- frame = self._source_traceback[-1]
- coro_repr += f', created at {frame[0]}:{frame[1]}'
-
- return f'<{self.__class__.__name__} {coro_repr}>'
-
- def __iter__(self):
- return self
-
- def __next__(self):
- return self.gen.send(None)
-
- def send(self, value):
- return self.gen.send(value)
-
- def throw(self, type, value=None, traceback=None):
- return self.gen.throw(type, value, traceback)
-
- def close(self):
- return self.gen.close()
-
- @property
- def gi_frame(self):
- return self.gen.gi_frame
-
- @property
- def gi_running(self):
- return self.gen.gi_running
-
- @property
- def gi_code(self):
- return self.gen.gi_code
-
- def __await__(self):
- return self
-
- @property
- def gi_yieldfrom(self):
- return self.gen.gi_yieldfrom
-
- def __del__(self):
- # Be careful accessing self.gen.frame -- self.gen might not exist.
- gen = getattr(self, 'gen', None)
- frame = getattr(gen, 'gi_frame', None)
- if frame is not None and frame.f_lasti == -1:
- msg = f'{self!r} was never yielded from'
- tb = getattr(self, '_source_traceback', ())
- if tb:
- tb = ''.join(traceback.format_list(tb))
- msg += (f'\nCoroutine object created at '
- f'(most recent call last, truncated to '
- f'{constants.DEBUG_STACK_DEPTH} last lines):\n')
- msg += tb.rstrip()
- logger.error(msg)
-
-
-def coroutine(func):
- """Decorator to mark coroutines.
-
- If the coroutine is not yielded from before it is destroyed,
- an error message is logged.
- """
- warnings.warn('"@coroutine" decorator is deprecated since Python 3.8, use "async def" instead',
- DeprecationWarning,
- stacklevel=2)
- if inspect.iscoroutinefunction(func):
- # In Python 3.5 that's all we need to do for coroutines
- # defined with "async def".
- return func
-
- if inspect.isgeneratorfunction(func):
- coro = func
- else:
- @functools.wraps(func)
- def coro(*args, **kw):
- res = func(*args, **kw)
- if (base_futures.isfuture(res) or inspect.isgenerator(res) or
- isinstance(res, CoroWrapper)):
- res = yield from res
- else:
- # If 'res' is an awaitable, run it.
- try:
- await_meth = res.__await__
- except AttributeError:
- pass
- else:
- if isinstance(res, collections.abc.Awaitable):
- res = yield from await_meth()
- return res
-
- coro = types.coroutine(coro)
- if not _DEBUG:
- wrapper = coro
- else:
- @functools.wraps(func)
- def wrapper(*args, **kwds):
- w = CoroWrapper(coro(*args, **kwds), func=func)
- if w._source_traceback:
- del w._source_traceback[-1]
- # Python < 3.5 does not implement __qualname__
- # on generator objects, so we set it manually.
- # We use getattr as some callables (such as
- # functools.partial may lack __qualname__).
- w.__name__ = getattr(func, '__name__', None)
- w.__qualname__ = getattr(func, '__qualname__', None)
- return w
-
- wrapper._is_coroutine = _is_coroutine # For iscoroutinefunction().
- return wrapper
-
-
# A marker for iscoroutinefunction.
_is_coroutine = object()
@@ -170,7 +27,7 @@
# Prioritize native coroutine check to speed-up
# asyncio.iscoroutine.
_COROUTINE_TYPES = (types.CoroutineType, types.GeneratorType,
- collections.abc.Coroutine, CoroWrapper)
+ collections.abc.Coroutine)
_iscoroutine_typecache = set()
@@ -193,16 +50,11 @@
def _format_coroutine(coro):
assert iscoroutine(coro)
- is_corowrapper = isinstance(coro, CoroWrapper)
-
def get_name(coro):
# Coroutines compiled with Cython sometimes don't have
# proper __qualname__ or __name__. While that is a bug
# in Cython, asyncio shouldn't crash with an AttributeError
# in its __repr__ functions.
- if is_corowrapper:
- return format_helpers._format_callback(coro.func, (), {})
-
if hasattr(coro, '__qualname__') and coro.__qualname__:
coro_name = coro.__qualname__
elif hasattr(coro, '__name__') and coro.__name__:
@@ -247,18 +99,8 @@
filename = coro_code.co_filename or '<empty co_filename>'
lineno = 0
- if (is_corowrapper and
- coro.func is not None and
- not inspect.isgeneratorfunction(coro.func)):
- source = format_helpers._get_function_source(coro.func)
- if source is not None:
- filename, lineno = source
- if coro_frame is None:
- coro_repr = f'{coro_name} done, defined at {filename}:{lineno}'
- else:
- coro_repr = f'{coro_name} running, defined at {filename}:{lineno}'
- elif coro_frame is not None:
+ if coro_frame is not None:
lineno = coro_frame.f_lineno
coro_repr = f'{coro_name} running at {filename}:{lineno}'
diff --git a/common/py3-stdlib/asyncio/events.py b/common/py3-stdlib/asyncio/events.py
index e088230..b179932 100644
--- a/common/py3-stdlib/asyncio/events.py
+++ b/common/py3-stdlib/asyncio/events.py
@@ -101,7 +101,6 @@
__slots__ = ['_scheduled', '_when']
def __init__(self, when, callback, args, loop, context=None):
- assert when is not None
super().__init__(callback, args, loop, context)
if self._source_traceback:
del self._source_traceback[-1]
@@ -275,7 +274,7 @@
# Method scheduling a coroutine object: create a task.
- def create_task(self, coro, *, name=None):
+ def create_task(self, coro, *, name=None, context=None):
raise NotImplementedError
# Methods for interacting with threads.
@@ -304,6 +303,7 @@
flags=0, sock=None, local_addr=None,
server_hostname=None,
ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None,
happy_eyeballs_delay=None, interleave=None):
raise NotImplementedError
@@ -313,6 +313,7 @@
flags=socket.AI_PASSIVE, sock=None, backlog=100,
ssl=None, reuse_address=None, reuse_port=None,
ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None,
start_serving=True):
"""A coroutine which creates a TCP server bound to host and port.
@@ -353,6 +354,10 @@
will wait for completion of the SSL handshake before aborting the
connection. Default is 60s.
+ ssl_shutdown_timeout is the time in seconds that an SSL server
+ will wait for completion of the SSL shutdown procedure
+ before aborting the connection. Default is 30s.
+
start_serving set to True (default) causes the created server
to start accepting connections immediately. When set to False,
the user should await Server.start_serving() or Server.serve_forever()
@@ -371,7 +376,8 @@
async def start_tls(self, transport, protocol, sslcontext, *,
server_side=False,
server_hostname=None,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
"""Upgrade a transport to TLS.
Return a new transport that *protocol* should start using
@@ -383,13 +389,15 @@
self, protocol_factory, path=None, *,
ssl=None, sock=None,
server_hostname=None,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
raise NotImplementedError
async def create_unix_server(
self, protocol_factory, path=None, *,
sock=None, backlog=100, ssl=None,
ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None,
start_serving=True):
"""A coroutine which creates a UNIX Domain Socket server.
@@ -411,6 +419,9 @@
ssl_handshake_timeout is the time in seconds that an SSL server
will wait for the SSL handshake to complete (defaults to 60s).
+ ssl_shutdown_timeout is the time in seconds that an SSL server
+ will wait for the SSL shutdown to finish (defaults to 30s).
+
start_serving set to True (default) causes the created server
to start accepting connections immediately. When set to False,
the user should await Server.start_serving() or Server.serve_forever()
@@ -421,7 +432,8 @@
async def connect_accepted_socket(
self, protocol_factory, sock,
*, ssl=None,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
"""Handle an accepted connection.
This is used by servers that accept connections outside of
@@ -534,9 +546,18 @@
async def sock_recv_into(self, sock, buf):
raise NotImplementedError
+ async def sock_recvfrom(self, sock, bufsize):
+ raise NotImplementedError
+
+ async def sock_recvfrom_into(self, sock, buf, nbytes=0):
+ raise NotImplementedError
+
async def sock_sendall(self, sock, data):
raise NotImplementedError
+ async def sock_sendto(self, sock, data, address):
+ raise NotImplementedError
+
async def sock_connect(self, sock, address):
raise NotImplementedError
@@ -661,7 +682,8 @@
def set_event_loop(self, loop):
"""Set the event loop."""
self._local._set_called = True
- assert loop is None or isinstance(loop, AbstractEventLoop)
+ if loop is not None and not isinstance(loop, AbstractEventLoop):
+ raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'")
self._local._loop = loop
def new_event_loop(self):
@@ -745,7 +767,8 @@
If policy is None, the default policy is restored."""
global _event_loop_policy
- assert policy is None or isinstance(policy, AbstractEventLoopPolicy)
+ if policy is not None and not isinstance(policy, AbstractEventLoopPolicy):
+ raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'")
_event_loop_policy = policy
diff --git a/common/py3-stdlib/asyncio/exceptions.py b/common/py3-stdlib/asyncio/exceptions.py
index f07e448..5ece595 100644
--- a/common/py3-stdlib/asyncio/exceptions.py
+++ b/common/py3-stdlib/asyncio/exceptions.py
@@ -1,7 +1,8 @@
"""asyncio exceptions."""
-__all__ = ('CancelledError', 'InvalidStateError', 'TimeoutError',
+__all__ = ('BrokenBarrierError',
+ 'CancelledError', 'InvalidStateError', 'TimeoutError',
'IncompleteReadError', 'LimitOverrunError',
'SendfileNotAvailableError')
@@ -10,8 +11,7 @@
"""The Future or Task was cancelled."""
-class TimeoutError(Exception):
- """The operation exceeded the given deadline."""
+TimeoutError = TimeoutError # make local alias for the standard exception
class InvalidStateError(Exception):
@@ -56,3 +56,7 @@
def __reduce__(self):
return type(self), (self.args[0], self.consumed)
+
+
+class BrokenBarrierError(RuntimeError):
+ """Barrier is broken by barrier.abort() call."""
diff --git a/common/py3-stdlib/asyncio/futures.py b/common/py3-stdlib/asyncio/futures.py
index 5d00ab4..3a6b44a 100644
--- a/common/py3-stdlib/asyncio/futures.py
+++ b/common/py3-stdlib/asyncio/futures.py
@@ -85,11 +85,8 @@
self._source_traceback = format_helpers.extract_stack(
sys._getframe(1))
- _repr_info = base_futures._future_repr_info
-
def __repr__(self):
- return '<{} {}>'.format(self.__class__.__name__,
- ' '.join(self._repr_info()))
+ return base_futures._future_repr(self)
def __del__(self):
if not self.__log_traceback:
@@ -132,6 +129,11 @@
This should only be called once when handling a cancellation since
it erases the saved context exception value.
"""
+ if self._cancelled_exc is not None:
+ exc = self._cancelled_exc
+ self._cancelled_exc = None
+ return exc
+
if self._cancel_message is None:
exc = exceptions.CancelledError()
else:
diff --git a/common/py3-stdlib/asyncio/locks.py b/common/py3-stdlib/asyncio/locks.py
index e192159..fd41dfd 100644
--- a/common/py3-stdlib/asyncio/locks.py
+++ b/common/py3-stdlib/asyncio/locks.py
@@ -1,14 +1,15 @@
"""Synchronization primitives."""
-__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore')
+__all__ = ('Lock', 'Event', 'Condition', 'Semaphore',
+ 'BoundedSemaphore', 'Barrier')
import collections
+import enum
from . import exceptions
from . import mixins
from . import tasks
-
class _ContextManagerMixin:
async def __aenter__(self):
await self.acquire()
@@ -74,8 +75,7 @@
"""
- def __init__(self, *, loop=mixins._marker):
- super().__init__(loop=loop)
+ def __init__(self):
self._waiters = None
self._locked = False
@@ -164,8 +164,7 @@
false.
"""
- def __init__(self, *, loop=mixins._marker):
- super().__init__(loop=loop)
+ def __init__(self):
self._waiters = collections.deque()
self._value = False
@@ -227,8 +226,7 @@
A new Lock object is created and used as the underlying lock.
"""
- def __init__(self, lock=None, *, loop=mixins._marker):
- super().__init__(loop=loop)
+ def __init__(self, lock=None):
if lock is None:
lock = Lock()
@@ -345,8 +343,7 @@
ValueError is raised.
"""
- def __init__(self, value=1, *, loop=mixins._marker):
- super().__init__(loop=loop)
+ def __init__(self, value=1):
if value < 0:
raise ValueError("Semaphore initial value must be >= 0")
self._waiters = None
@@ -428,11 +425,163 @@
above the initial value.
"""
- def __init__(self, value=1, *, loop=mixins._marker):
+ def __init__(self, value=1):
self._bound_value = value
- super().__init__(value, loop=loop)
+ super().__init__(value)
def release(self):
if self._value >= self._bound_value:
raise ValueError('BoundedSemaphore released too many times')
super().release()
+
+
+
+class _BarrierState(enum.Enum):
+ FILLING = 'filling'
+ DRAINING = 'draining'
+ RESETTING = 'resetting'
+ BROKEN = 'broken'
+
+
+class Barrier(mixins._LoopBoundMixin):
+ """Asyncio equivalent to threading.Barrier
+
+ Implements a Barrier primitive.
+ Useful for synchronizing a fixed number of tasks at known synchronization
+ points. Tasks block on 'wait()' and are simultaneously awoken once they
+ have all made their call.
+ """
+
+ def __init__(self, parties):
+ """Create a barrier, initialised to 'parties' tasks."""
+ if parties < 1:
+ raise ValueError('parties must be > 0')
+
+ self._cond = Condition() # notify all tasks when state changes
+
+ self._parties = parties
+ self._state = _BarrierState.FILLING
+ self._count = 0 # count tasks in Barrier
+
+ def __repr__(self):
+ res = super().__repr__()
+ extra = f'{self._state.value}'
+ if not self.broken:
+ extra += f', waiters:{self.n_waiting}/{self.parties}'
+ return f'<{res[1:-1]} [{extra}]>'
+
+ async def __aenter__(self):
+ # wait for the barrier reaches the parties number
+ # when start draining release and return index of waited task
+ return await self.wait()
+
+ async def __aexit__(self, *args):
+ pass
+
+ async def wait(self):
+ """Wait for the barrier.
+
+ When the specified number of tasks have started waiting, they are all
+ simultaneously awoken.
+ Returns an unique and individual index number from 0 to 'parties-1'.
+ """
+ async with self._cond:
+ await self._block() # Block while the barrier drains or resets.
+ try:
+ index = self._count
+ self._count += 1
+ if index + 1 == self._parties:
+ # We release the barrier
+ await self._release()
+ else:
+ await self._wait()
+ return index
+ finally:
+ self._count -= 1
+ # Wake up any tasks waiting for barrier to drain.
+ self._exit()
+
+ async def _block(self):
+ # Block until the barrier is ready for us,
+ # or raise an exception if it is broken.
+ #
+ # It is draining or resetting, wait until done
+ # unless a CancelledError occurs
+ await self._cond.wait_for(
+ lambda: self._state not in (
+ _BarrierState.DRAINING, _BarrierState.RESETTING
+ )
+ )
+
+ # see if the barrier is in a broken state
+ if self._state is _BarrierState.BROKEN:
+ raise exceptions.BrokenBarrierError("Barrier aborted")
+
+ async def _release(self):
+ # Release the tasks waiting in the barrier.
+
+ # Enter draining state.
+ # Next waiting tasks will be blocked until the end of draining.
+ self._state = _BarrierState.DRAINING
+ self._cond.notify_all()
+
+ async def _wait(self):
+ # Wait in the barrier until we are released. Raise an exception
+ # if the barrier is reset or broken.
+
+ # wait for end of filling
+ # unless a CancelledError occurs
+ await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING)
+
+ if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING):
+ raise exceptions.BrokenBarrierError("Abort or reset of barrier")
+
+ def _exit(self):
+ # If we are the last tasks to exit the barrier, signal any tasks
+ # waiting for the barrier to drain.
+ if self._count == 0:
+ if self._state in (_BarrierState.RESETTING, _BarrierState.DRAINING):
+ self._state = _BarrierState.FILLING
+ self._cond.notify_all()
+
+ async def reset(self):
+ """Reset the barrier to the initial state.
+
+ Any tasks currently waiting will get the BrokenBarrier exception
+ raised.
+ """
+ async with self._cond:
+ if self._count > 0:
+ if self._state is not _BarrierState.RESETTING:
+ #reset the barrier, waking up tasks
+ self._state = _BarrierState.RESETTING
+ else:
+ self._state = _BarrierState.FILLING
+ self._cond.notify_all()
+
+ async def abort(self):
+ """Place the barrier into a 'broken' state.
+
+ Useful in case of error. Any currently waiting tasks and tasks
+ attempting to 'wait()' will have BrokenBarrierError raised.
+ """
+ async with self._cond:
+ self._state = _BarrierState.BROKEN
+ self._cond.notify_all()
+
+ @property
+ def parties(self):
+ """Return the number of tasks required to trip the barrier."""
+ return self._parties
+
+ @property
+ def n_waiting(self):
+ """Return the number of tasks currently waiting at the barrier."""
+ if self._state is _BarrierState.FILLING:
+ return self._count
+ return 0
+
+ @property
+ def broken(self):
+ """Return True if the barrier is in a broken state."""
+ return self._state is _BarrierState.BROKEN
diff --git a/common/py3-stdlib/asyncio/mixins.py b/common/py3-stdlib/asyncio/mixins.py
index 650df05..c6bf973 100644
--- a/common/py3-stdlib/asyncio/mixins.py
+++ b/common/py3-stdlib/asyncio/mixins.py
@@ -5,20 +5,10 @@
_global_lock = threading.Lock()
-# Used as a sentinel for loop parameter
-_marker = object()
-
class _LoopBoundMixin:
_loop = None
- def __init__(self, *, loop=_marker):
- if loop is not _marker:
- raise TypeError(
- f'As of 3.10, the *loop* parameter was removed from '
- f'{type(self).__name__}() since it is no longer necessary'
- )
-
def _get_loop(self):
loop = events._get_running_loop()
diff --git a/common/py3-stdlib/asyncio/proactor_events.py b/common/py3-stdlib/asyncio/proactor_events.py
index 0916d9e..c6aab40 100644
--- a/common/py3-stdlib/asyncio/proactor_events.py
+++ b/common/py3-stdlib/asyncio/proactor_events.py
@@ -225,7 +225,7 @@
length = self._pending_data_length
self._pending_data_length = -1
if length > -1:
- # Call the protocol methode after calling _loop_reading(),
+ # Call the protocol method after calling _loop_reading(),
# since the protocol can decide to pause reading again.
self._loop.call_soon(self._data_received, self._data[:length], length)
@@ -463,6 +463,7 @@
waiter=None, extra=None):
self._address = address
self._empty_waiter = None
+ self._buffer_size = 0
# We don't need to call _protocol.connection_made() since our base
# constructor does it for us.
super().__init__(loop, sock, protocol, waiter=waiter, extra=extra)
@@ -475,7 +476,7 @@
_set_socket_extra(self, sock)
def get_write_buffer_size(self):
- return sum(len(data) for data, _ in self._buffer)
+ return self._buffer_size
def abort(self):
self._force_close(None)
@@ -500,6 +501,7 @@
# Ensure that what we buffer is immutable.
self._buffer.append((bytes(data), addr))
+ self._buffer_size += len(data)
if self._write_fut is None:
# No current write operations are active, kick one off
@@ -526,6 +528,7 @@
return
data, addr = self._buffer.popleft()
+ self._buffer_size -= len(data)
if self._address is not None:
self._write_fut = self._loop._proactor.send(self._sock,
data)
@@ -647,11 +650,13 @@
self, rawsock, protocol, sslcontext, waiter=None,
*, server_side=False, server_hostname=None,
extra=None, server=None,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
ssl_protocol = sslproto.SSLProtocol(
self, protocol, sslcontext, waiter,
server_side, server_hostname,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout)
_ProactorSocketTransport(self, rawsock, ssl_protocol,
extra=extra, server=server)
return ssl_protocol._app_transport
@@ -702,9 +707,21 @@
async def sock_recv_into(self, sock, buf):
return await self._proactor.recv_into(sock, buf)
+ async def sock_recvfrom(self, sock, bufsize):
+ return await self._proactor.recvfrom(sock, bufsize)
+
+ async def sock_recvfrom_into(self, sock, buf, nbytes=0):
+ if not nbytes:
+ nbytes = len(buf)
+
+ return await self._proactor.recvfrom_into(sock, buf, nbytes)
+
async def sock_sendall(self, sock, data):
return await self._proactor.send(sock, data)
+ async def sock_sendto(self, sock, data, address):
+ return await self._proactor.sendto(sock, data, 0, address)
+
async def sock_connect(self, sock, address):
return await self._proactor.connect(sock, address)
@@ -817,7 +834,8 @@
def _start_serving(self, protocol_factory, sock,
sslcontext=None, server=None, backlog=100,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
def loop(f=None):
try:
@@ -831,7 +849,8 @@
self._make_ssl_transport(
conn, protocol, sslcontext, server_side=True,
extra={'peername': addr}, server=server,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout)
else:
self._make_socket_transport(
conn, protocol,
diff --git a/common/py3-stdlib/asyncio/queues.py b/common/py3-stdlib/asyncio/queues.py
index 10dd689..a9656a6 100644
--- a/common/py3-stdlib/asyncio/queues.py
+++ b/common/py3-stdlib/asyncio/queues.py
@@ -30,8 +30,7 @@
interrupted between calling qsize() and doing an operation on the Queue.
"""
- def __init__(self, maxsize=0, *, loop=mixins._marker):
- super().__init__(loop=loop)
+ def __init__(self, maxsize=0):
self._maxsize = maxsize
# Futures.
diff --git a/common/py3-stdlib/asyncio/runners.py b/common/py3-stdlib/asyncio/runners.py
index 9a5e9a4..b3e0c44 100644
--- a/common/py3-stdlib/asyncio/runners.py
+++ b/common/py3-stdlib/asyncio/runners.py
@@ -1,10 +1,162 @@
-__all__ = 'run',
+__all__ = ('Runner', 'run')
+import contextvars
+import enum
+import functools
+import threading
+import signal
+import sys
from . import coroutines
from . import events
+from . import exceptions
from . import tasks
+class _State(enum.Enum):
+ CREATED = "created"
+ INITIALIZED = "initialized"
+ CLOSED = "closed"
+
+
+class Runner:
+ """A context manager that controls event loop life cycle.
+
+ The context manager always creates a new event loop,
+ allows to run async functions inside it,
+ and properly finalizes the loop at the context manager exit.
+
+ If debug is True, the event loop will be run in debug mode.
+ If loop_factory is passed, it is used for new event loop creation.
+
+ asyncio.run(main(), debug=True)
+
+ is a shortcut for
+
+ with asyncio.Runner(debug=True) as runner:
+ runner.run(main())
+
+ The run() method can be called multiple times within the runner's context.
+
+ This can be useful for interactive console (e.g. IPython),
+ unittest runners, console tools, -- everywhere when async code
+ is called from existing sync framework and where the preferred single
+ asyncio.run() call doesn't work.
+
+ """
+
+ # Note: the class is final, it is not intended for inheritance.
+
+ def __init__(self, *, debug=None, loop_factory=None):
+ self._state = _State.CREATED
+ self._debug = debug
+ self._loop_factory = loop_factory
+ self._loop = None
+ self._context = None
+ self._interrupt_count = 0
+ self._set_event_loop = False
+
+ def __enter__(self):
+ self._lazy_init()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ def close(self):
+ """Shutdown and close event loop."""
+ if self._state is not _State.INITIALIZED:
+ return
+ try:
+ loop = self._loop
+ _cancel_all_tasks(loop)
+ loop.run_until_complete(loop.shutdown_asyncgens())
+ loop.run_until_complete(loop.shutdown_default_executor())
+ finally:
+ if self._set_event_loop:
+ events.set_event_loop(None)
+ loop.close()
+ self._loop = None
+ self._state = _State.CLOSED
+
+ def get_loop(self):
+ """Return embedded event loop."""
+ self._lazy_init()
+ return self._loop
+
+ def run(self, coro, *, context=None):
+ """Run a coroutine inside the embedded event loop."""
+ if not coroutines.iscoroutine(coro):
+ raise ValueError("a coroutine was expected, got {!r}".format(coro))
+
+ if events._get_running_loop() is not None:
+ # fail fast with short traceback
+ raise RuntimeError(
+ "Runner.run() cannot be called from a running event loop")
+
+ self._lazy_init()
+
+ if context is None:
+ context = self._context
+ task = self._loop.create_task(coro, context=context)
+
+ if (threading.current_thread() is threading.main_thread()
+ and signal.getsignal(signal.SIGINT) is signal.default_int_handler
+ ):
+ sigint_handler = functools.partial(self._on_sigint, main_task=task)
+ try:
+ signal.signal(signal.SIGINT, sigint_handler)
+ except ValueError:
+ # `signal.signal` may throw if `threading.main_thread` does
+ # not support signals (e.g. embedded interpreter with signals
+ # not registered - see gh-91880)
+ sigint_handler = None
+ else:
+ sigint_handler = None
+
+ self._interrupt_count = 0
+ try:
+ return self._loop.run_until_complete(task)
+ except exceptions.CancelledError:
+ if self._interrupt_count > 0:
+ uncancel = getattr(task, "uncancel", None)
+ if uncancel is not None and uncancel() == 0:
+ raise KeyboardInterrupt()
+ raise # CancelledError
+ finally:
+ if (sigint_handler is not None
+ and signal.getsignal(signal.SIGINT) is sigint_handler
+ ):
+ signal.signal(signal.SIGINT, signal.default_int_handler)
+
+ def _lazy_init(self):
+ if self._state is _State.CLOSED:
+ raise RuntimeError("Runner is closed")
+ if self._state is _State.INITIALIZED:
+ return
+ if self._loop_factory is None:
+ self._loop = events.new_event_loop()
+ if not self._set_event_loop:
+ # Call set_event_loop only once to avoid calling
+ # attach_loop multiple times on child watchers
+ events.set_event_loop(self._loop)
+ self._set_event_loop = True
+ else:
+ self._loop = self._loop_factory()
+ if self._debug is not None:
+ self._loop.set_debug(self._debug)
+ self._context = contextvars.copy_context()
+ self._state = _State.INITIALIZED
+
+ def _on_sigint(self, signum, frame, main_task):
+ self._interrupt_count += 1
+ if self._interrupt_count == 1 and not main_task.done():
+ main_task.cancel()
+ # wakeup loop if it is blocked by select() with long timeout
+ self._loop.call_soon_threadsafe(lambda: None)
+ return
+ raise KeyboardInterrupt()
+
+
def run(main, *, debug=None):
"""Execute the coroutine and return the result.
@@ -30,26 +182,12 @@
asyncio.run(main())
"""
if events._get_running_loop() is not None:
+ # fail fast with short traceback
raise RuntimeError(
"asyncio.run() cannot be called from a running event loop")
- if not coroutines.iscoroutine(main):
- raise ValueError("a coroutine was expected, got {!r}".format(main))
-
- loop = events.new_event_loop()
- try:
- events.set_event_loop(loop)
- if debug is not None:
- loop.set_debug(debug)
- return loop.run_until_complete(main)
- finally:
- try:
- _cancel_all_tasks(loop)
- loop.run_until_complete(loop.shutdown_asyncgens())
- loop.run_until_complete(loop.shutdown_default_executor())
- finally:
- events.set_event_loop(None)
- loop.close()
+ with Runner(debug=debug) as runner:
+ return runner.run(main)
def _cancel_all_tasks(loop):
diff --git a/common/py3-stdlib/asyncio/selector_events.py b/common/py3-stdlib/asyncio/selector_events.py
index 8282f28..96e61f7 100644
--- a/common/py3-stdlib/asyncio/selector_events.py
+++ b/common/py3-stdlib/asyncio/selector_events.py
@@ -65,11 +65,15 @@
self, rawsock, protocol, sslcontext, waiter=None,
*, server_side=False, server_hostname=None,
extra=None, server=None,
- ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT):
+ ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
+ ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT,
+ ):
ssl_protocol = sslproto.SSLProtocol(
- self, protocol, sslcontext, waiter,
- server_side, server_hostname,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ self, protocol, sslcontext, waiter,
+ server_side, server_hostname,
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout
+ )
_SelectorSocketTransport(self, rawsock, ssl_protocol,
extra=extra, server=server)
return ssl_protocol._app_transport
@@ -141,15 +145,17 @@
def _start_serving(self, protocol_factory, sock,
sslcontext=None, server=None, backlog=100,
- ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT):
+ ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
+ ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
self._add_reader(sock.fileno(), self._accept_connection,
protocol_factory, sock, sslcontext, server, backlog,
- ssl_handshake_timeout)
+ ssl_handshake_timeout, ssl_shutdown_timeout)
def _accept_connection(
self, protocol_factory, sock,
sslcontext=None, server=None, backlog=100,
- ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT):
+ ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
+ ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
# This method is only called once for each event loop tick where the
# listening socket has triggered an EVENT_READ. There may be multiple
# connections waiting for an .accept() so it is called in a loop.
@@ -180,20 +186,22 @@
self.call_later(constants.ACCEPT_RETRY_DELAY,
self._start_serving,
protocol_factory, sock, sslcontext, server,
- backlog, ssl_handshake_timeout)
+ backlog, ssl_handshake_timeout,
+ ssl_shutdown_timeout)
else:
raise # The event loop will catch, log and ignore it.
else:
extra = {'peername': addr}
accept = self._accept_connection2(
protocol_factory, conn, extra, sslcontext, server,
- ssl_handshake_timeout)
+ ssl_handshake_timeout, ssl_shutdown_timeout)
self.create_task(accept)
async def _accept_connection2(
self, protocol_factory, conn, extra,
sslcontext=None, server=None,
- ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT):
+ ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
+ ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
protocol = None
transport = None
try:
@@ -203,7 +211,8 @@
transport = self._make_ssl_transport(
conn, protocol, sslcontext, waiter=waiter,
server_side=True, extra=extra, server=server,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout)
else:
transport = self._make_socket_transport(
conn, protocol, waiter=waiter, extra=extra,
@@ -425,6 +434,88 @@
else:
fut.set_result(nbytes)
+ async def sock_recvfrom(self, sock, bufsize):
+ """Receive a datagram from a datagram socket.
+
+ The return value is a tuple of (bytes, address) representing the
+ datagram received and the address it came from.
+ The maximum amount of data to be received at once is specified by
+ nbytes.
+ """
+ base_events._check_ssl_socket(sock)
+ if self._debug and sock.gettimeout() != 0:
+ raise ValueError("the socket must be non-blocking")
+ try:
+ return sock.recvfrom(bufsize)
+ except (BlockingIOError, InterruptedError):
+ pass
+ fut = self.create_future()
+ fd = sock.fileno()
+ self._ensure_fd_no_transport(fd)
+ handle = self._add_reader(fd, self._sock_recvfrom, fut, sock, bufsize)
+ fut.add_done_callback(
+ functools.partial(self._sock_read_done, fd, handle=handle))
+ return await fut
+
+ def _sock_recvfrom(self, fut, sock, bufsize):
+ # _sock_recvfrom() can add itself as an I/O callback if the operation
+ # can't be done immediately. Don't use it directly, call
+ # sock_recvfrom().
+ if fut.done():
+ return
+ try:
+ result = sock.recvfrom(bufsize)
+ except (BlockingIOError, InterruptedError):
+ return # try again next time
+ except (SystemExit, KeyboardInterrupt):
+ raise
+ except BaseException as exc:
+ fut.set_exception(exc)
+ else:
+ fut.set_result(result)
+
+ async def sock_recvfrom_into(self, sock, buf, nbytes=0):
+ """Receive data from the socket.
+
+ The received data is written into *buf* (a writable buffer).
+ The return value is a tuple of (number of bytes written, address).
+ """
+ base_events._check_ssl_socket(sock)
+ if self._debug and sock.gettimeout() != 0:
+ raise ValueError("the socket must be non-blocking")
+ if not nbytes:
+ nbytes = len(buf)
+
+ try:
+ return sock.recvfrom_into(buf, nbytes)
+ except (BlockingIOError, InterruptedError):
+ pass
+ fut = self.create_future()
+ fd = sock.fileno()
+ self._ensure_fd_no_transport(fd)
+ handle = self._add_reader(fd, self._sock_recvfrom_into, fut, sock, buf,
+ nbytes)
+ fut.add_done_callback(
+ functools.partial(self._sock_read_done, fd, handle=handle))
+ return await fut
+
+ def _sock_recvfrom_into(self, fut, sock, buf, bufsize):
+ # _sock_recv_into() can add itself as an I/O callback if the operation
+ # can't be done immediately. Don't use it directly, call
+ # sock_recv_into().
+ if fut.done():
+ return
+ try:
+ result = sock.recvfrom_into(buf, bufsize)
+ except (BlockingIOError, InterruptedError):
+ return # try again next time
+ except (SystemExit, KeyboardInterrupt):
+ raise
+ except BaseException as exc:
+ fut.set_exception(exc)
+ else:
+ fut.set_result(result)
+
async def sock_sendall(self, sock, data):
"""Send data to the socket.
@@ -478,6 +569,48 @@
else:
pos[0] = start
+ async def sock_sendto(self, sock, data, address):
+ """Send data to the socket.
+
+ The socket must be connected to a remote socket. This method continues
+ to send data from data until either all data has been sent or an
+ error occurs. None is returned on success. On error, an exception is
+ raised, and there is no way to determine how much data, if any, was
+ successfully processed by the receiving end of the connection.
+ """
+ base_events._check_ssl_socket(sock)
+ if self._debug and sock.gettimeout() != 0:
+ raise ValueError("the socket must be non-blocking")
+ try:
+ return sock.sendto(data, address)
+ except (BlockingIOError, InterruptedError):
+ pass
+
+ fut = self.create_future()
+ fd = sock.fileno()
+ self._ensure_fd_no_transport(fd)
+ # use a trick with a list in closure to store a mutable state
+ handle = self._add_writer(fd, self._sock_sendto, fut, sock, data,
+ address)
+ fut.add_done_callback(
+ functools.partial(self._sock_write_done, fd, handle=handle))
+ return await fut
+
+ def _sock_sendto(self, fut, sock, data, address):
+ if fut.done():
+ # Future cancellation can be scheduled on previous loop iteration
+ return
+ try:
+ n = sock.sendto(data, 0, address)
+ except (BlockingIOError, InterruptedError):
+ return
+ except (SystemExit, KeyboardInterrupt):
+ raise
+ except BaseException as exc:
+ fut.set_exception(exc)
+ else:
+ fut.set_result(n)
+
async def sock_connect(self, sock, address):
"""Connect to a remote socket at address.
@@ -650,6 +783,8 @@
self._buffer = self._buffer_factory()
self._conn_lost = 0 # Set when call to connection_lost scheduled.
self._closing = False # Set when close() called.
+ self._paused = False # Set when pause_reading() called
+
if self._server is not None:
self._server._attach()
loop._transports[self._sock_fd] = self
@@ -695,6 +830,25 @@
def is_closing(self):
return self._closing
+ def is_reading(self):
+ return not self.is_closing() and not self._paused
+
+ def pause_reading(self):
+ if not self.is_reading():
+ return
+ self._paused = True
+ self._loop._remove_reader(self._sock_fd)
+ if self._loop.get_debug():
+ logger.debug("%r pauses reading", self)
+
+ def resume_reading(self):
+ if self._closing or not self._paused:
+ return
+ self._paused = False
+ self._add_reader(self._sock_fd, self._read_ready)
+ if self._loop.get_debug():
+ logger.debug("%r resumes reading", self)
+
def close(self):
if self._closing:
return
@@ -754,9 +908,8 @@
return len(self._buffer)
def _add_reader(self, fd, callback, *args):
- if self._closing:
+ if not self.is_reading():
return
-
self._loop._add_reader(fd, callback, *args)
@@ -771,7 +924,6 @@
self._read_ready_cb = None
super().__init__(loop, sock, protocol, extra, server)
self._eof = False
- self._paused = False
self._empty_waiter = None
# Disable the Nagle algorithm -- small writes will be
@@ -796,25 +948,6 @@
super().set_protocol(protocol)
- def is_reading(self):
- return not self._paused and not self._closing
-
- def pause_reading(self):
- if self._closing or self._paused:
- return
- self._paused = True
- self._loop._remove_reader(self._sock_fd)
- if self._loop.get_debug():
- logger.debug("%r pauses reading", self)
-
- def resume_reading(self):
- if self._closing or not self._paused:
- return
- self._paused = False
- self._add_reader(self._sock_fd, self._read_ready)
- if self._loop.get_debug():
- logger.debug("%r resumes reading", self)
-
def _read_ready(self):
self._read_ready_cb()
@@ -1006,6 +1139,7 @@
waiter=None, extra=None):
super().__init__(loop, sock, protocol, extra)
self._address = address
+ self._buffer_size = 0
self._loop.call_soon(self._protocol.connection_made, self)
# only start reading when connection_made() has been called
self._loop.call_soon(self._add_reader,
@@ -1016,7 +1150,7 @@
waiter, None)
def get_write_buffer_size(self):
- return sum(len(data) for data, _ in self._buffer)
+ return self._buffer_size
def _read_ready(self):
if self._conn_lost:
@@ -1075,11 +1209,13 @@
# Ensure that what we buffer is immutable.
self._buffer.append((bytes(data), addr))
+ self._buffer_size += len(data)
self._maybe_pause_protocol()
def _sendto_ready(self):
while self._buffer:
data, addr = self._buffer.popleft()
+ self._buffer_size -= len(data)
try:
if self._extra['peername']:
self._sock.send(data)
@@ -1087,6 +1223,7 @@
self._sock.sendto(data, addr)
except (BlockingIOError, InterruptedError):
self._buffer.appendleft((data, addr)) # Try again later.
+ self._buffer_size += len(data)
break
except OSError as exc:
self._protocol.error_received(exc)
diff --git a/common/py3-stdlib/asyncio/sslproto.py b/common/py3-stdlib/asyncio/sslproto.py
index 00fc16c..488e17d 100644
--- a/common/py3-stdlib/asyncio/sslproto.py
+++ b/common/py3-stdlib/asyncio/sslproto.py
@@ -1,4 +1,5 @@
import collections
+import enum
import warnings
try:
import ssl
@@ -6,10 +7,38 @@
ssl = None
from . import constants
+from . import exceptions
from . import protocols
from . import transports
from .log import logger
+if ssl is not None:
+ SSLAgainErrors = (ssl.SSLWantReadError, ssl.SSLSyscallError)
+
+
+class SSLProtocolState(enum.Enum):
+ UNWRAPPED = "UNWRAPPED"
+ DO_HANDSHAKE = "DO_HANDSHAKE"
+ WRAPPED = "WRAPPED"
+ FLUSHING = "FLUSHING"
+ SHUTDOWN = "SHUTDOWN"
+
+
+class AppProtocolState(enum.Enum):
+ # This tracks the state of app protocol (https://git.io/fj59P):
+ #
+ # INIT -cm-> CON_MADE [-dr*->] [-er-> EOF?] -cl-> CON_LOST
+ #
+ # * cm: connection_made()
+ # * dr: data_received()
+ # * er: eof_received()
+ # * cl: connection_lost()
+
+ STATE_INIT = "STATE_INIT"
+ STATE_CON_MADE = "STATE_CON_MADE"
+ STATE_EOF = "STATE_EOF"
+ STATE_CON_LOST = "STATE_CON_LOST"
+
def _create_transport_context(server_side, server_hostname):
if server_side:
@@ -25,269 +54,35 @@
return sslcontext
-# States of an _SSLPipe.
-_UNWRAPPED = "UNWRAPPED"
-_DO_HANDSHAKE = "DO_HANDSHAKE"
-_WRAPPED = "WRAPPED"
-_SHUTDOWN = "SHUTDOWN"
+def add_flowcontrol_defaults(high, low, kb):
+ if high is None:
+ if low is None:
+ hi = kb * 1024
+ else:
+ lo = low
+ hi = 4 * lo
+ else:
+ hi = high
+ if low is None:
+ lo = hi // 4
+ else:
+ lo = low
+ if not hi >= lo >= 0:
+ raise ValueError('high (%r) must be >= low (%r) must be >= 0' %
+ (hi, lo))
-class _SSLPipe(object):
- """An SSL "Pipe".
-
- An SSL pipe allows you to communicate with an SSL/TLS protocol instance
- through memory buffers. It can be used to implement a security layer for an
- existing connection where you don't have access to the connection's file
- descriptor, or for some reason you don't want to use it.
-
- An SSL pipe can be in "wrapped" and "unwrapped" mode. In unwrapped mode,
- data is passed through untransformed. In wrapped mode, application level
- data is encrypted to SSL record level data and vice versa. The SSL record
- level is the lowest level in the SSL protocol suite and is what travels
- as-is over the wire.
-
- An SslPipe initially is in "unwrapped" mode. To start SSL, call
- do_handshake(). To shutdown SSL again, call unwrap().
- """
-
- max_size = 256 * 1024 # Buffer size passed to read()
-
- def __init__(self, context, server_side, server_hostname=None):
- """
- The *context* argument specifies the ssl.SSLContext to use.
-
- The *server_side* argument indicates whether this is a server side or
- client side transport.
-
- The optional *server_hostname* argument can be used to specify the
- hostname you are connecting to. You may only specify this parameter if
- the _ssl module supports Server Name Indication (SNI).
- """
- self._context = context
- self._server_side = server_side
- self._server_hostname = server_hostname
- self._state = _UNWRAPPED
- self._incoming = ssl.MemoryBIO()
- self._outgoing = ssl.MemoryBIO()
- self._sslobj = None
- self._need_ssldata = False
- self._handshake_cb = None
- self._shutdown_cb = None
-
- @property
- def context(self):
- """The SSL context passed to the constructor."""
- return self._context
-
- @property
- def ssl_object(self):
- """The internal ssl.SSLObject instance.
-
- Return None if the pipe is not wrapped.
- """
- return self._sslobj
-
- @property
- def need_ssldata(self):
- """Whether more record level data is needed to complete a handshake
- that is currently in progress."""
- return self._need_ssldata
-
- @property
- def wrapped(self):
- """
- Whether a security layer is currently in effect.
-
- Return False during handshake.
- """
- return self._state == _WRAPPED
-
- def do_handshake(self, callback=None):
- """Start the SSL handshake.
-
- Return a list of ssldata. A ssldata element is a list of buffers
-
- The optional *callback* argument can be used to install a callback that
- will be called when the handshake is complete. The callback will be
- called with None if successful, else an exception instance.
- """
- if self._state != _UNWRAPPED:
- raise RuntimeError('handshake in progress or completed')
- self._sslobj = self._context.wrap_bio(
- self._incoming, self._outgoing,
- server_side=self._server_side,
- server_hostname=self._server_hostname)
- self._state = _DO_HANDSHAKE
- self._handshake_cb = callback
- ssldata, appdata = self.feed_ssldata(b'', only_handshake=True)
- assert len(appdata) == 0
- return ssldata
-
- def shutdown(self, callback=None):
- """Start the SSL shutdown sequence.
-
- Return a list of ssldata. A ssldata element is a list of buffers
-
- The optional *callback* argument can be used to install a callback that
- will be called when the shutdown is complete. The callback will be
- called without arguments.
- """
- if self._state == _UNWRAPPED:
- raise RuntimeError('no security layer present')
- if self._state == _SHUTDOWN:
- raise RuntimeError('shutdown in progress')
- assert self._state in (_WRAPPED, _DO_HANDSHAKE)
- self._state = _SHUTDOWN
- self._shutdown_cb = callback
- ssldata, appdata = self.feed_ssldata(b'')
- assert appdata == [] or appdata == [b'']
- return ssldata
-
- def feed_eof(self):
- """Send a potentially "ragged" EOF.
-
- This method will raise an SSL_ERROR_EOF exception if the EOF is
- unexpected.
- """
- self._incoming.write_eof()
- ssldata, appdata = self.feed_ssldata(b'')
- assert appdata == [] or appdata == [b'']
-
- def feed_ssldata(self, data, only_handshake=False):
- """Feed SSL record level data into the pipe.
-
- The data must be a bytes instance. It is OK to send an empty bytes
- instance. This can be used to get ssldata for a handshake initiated by
- this endpoint.
-
- Return a (ssldata, appdata) tuple. The ssldata element is a list of
- buffers containing SSL data that needs to be sent to the remote SSL.
-
- The appdata element is a list of buffers containing plaintext data that
- needs to be forwarded to the application. The appdata list may contain
- an empty buffer indicating an SSL "close_notify" alert. This alert must
- be acknowledged by calling shutdown().
- """
- if self._state == _UNWRAPPED:
- # If unwrapped, pass plaintext data straight through.
- if data:
- appdata = [data]
- else:
- appdata = []
- return ([], appdata)
-
- self._need_ssldata = False
- if data:
- self._incoming.write(data)
-
- ssldata = []
- appdata = []
- try:
- if self._state == _DO_HANDSHAKE:
- # Call do_handshake() until it doesn't raise anymore.
- self._sslobj.do_handshake()
- self._state = _WRAPPED
- if self._handshake_cb:
- self._handshake_cb(None)
- if only_handshake:
- return (ssldata, appdata)
- # Handshake done: execute the wrapped block
-
- if self._state == _WRAPPED:
- # Main state: read data from SSL until close_notify
- while True:
- chunk = self._sslobj.read(self.max_size)
- appdata.append(chunk)
- if not chunk: # close_notify
- break
-
- elif self._state == _SHUTDOWN:
- # Call shutdown() until it doesn't raise anymore.
- self._sslobj.unwrap()
- self._sslobj = None
- self._state = _UNWRAPPED
- if self._shutdown_cb:
- self._shutdown_cb()
-
- elif self._state == _UNWRAPPED:
- # Drain possible plaintext data after close_notify.
- appdata.append(self._incoming.read())
- except (ssl.SSLError, ssl.CertificateError) as exc:
- exc_errno = getattr(exc, 'errno', None)
- if exc_errno not in (
- ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE,
- ssl.SSL_ERROR_SYSCALL):
- if self._state == _DO_HANDSHAKE and self._handshake_cb:
- self._handshake_cb(exc)
- raise
- self._need_ssldata = (exc_errno == ssl.SSL_ERROR_WANT_READ)
-
- # Check for record level data that needs to be sent back.
- # Happens for the initial handshake and renegotiations.
- if self._outgoing.pending:
- ssldata.append(self._outgoing.read())
- return (ssldata, appdata)
-
- def feed_appdata(self, data, offset=0):
- """Feed plaintext data into the pipe.
-
- Return an (ssldata, offset) tuple. The ssldata element is a list of
- buffers containing record level data that needs to be sent to the
- remote SSL instance. The offset is the number of plaintext bytes that
- were processed, which may be less than the length of data.
-
- NOTE: In case of short writes, this call MUST be retried with the SAME
- buffer passed into the *data* argument (i.e. the id() must be the
- same). This is an OpenSSL requirement. A further particularity is that
- a short write will always have offset == 0, because the _ssl module
- does not enable partial writes. And even though the offset is zero,
- there will still be encrypted data in ssldata.
- """
- assert 0 <= offset <= len(data)
- if self._state == _UNWRAPPED:
- # pass through data in unwrapped mode
- if offset < len(data):
- ssldata = [data[offset:]]
- else:
- ssldata = []
- return (ssldata, len(data))
-
- ssldata = []
- view = memoryview(data)
- while True:
- self._need_ssldata = False
- try:
- if offset < len(view):
- offset += self._sslobj.write(view[offset:])
- except ssl.SSLError as exc:
- # It is not allowed to call write() after unwrap() until the
- # close_notify is acknowledged. We return the condition to the
- # caller as a short write.
- exc_errno = getattr(exc, 'errno', None)
- if exc.reason == 'PROTOCOL_IS_SHUTDOWN':
- exc_errno = exc.errno = ssl.SSL_ERROR_WANT_READ
- if exc_errno not in (ssl.SSL_ERROR_WANT_READ,
- ssl.SSL_ERROR_WANT_WRITE,
- ssl.SSL_ERROR_SYSCALL):
- raise
- self._need_ssldata = (exc_errno == ssl.SSL_ERROR_WANT_READ)
-
- # See if there's any record level data back for us.
- if self._outgoing.pending:
- ssldata.append(self._outgoing.read())
- if offset == len(view) or self._need_ssldata:
- break
- return (ssldata, offset)
+ return hi, lo
class _SSLProtocolTransport(transports._FlowControlMixin,
transports.Transport):
+ _start_tls_compatible = True
_sendfile_compatible = constants._SendfileMode.FALLBACK
def __init__(self, loop, ssl_protocol):
self._loop = loop
- # SSLProtocol instance
self._ssl_protocol = ssl_protocol
self._closed = False
@@ -312,19 +107,21 @@
protocol's connection_lost() method will (eventually) called
with None as its argument.
"""
- self._closed = True
- self._ssl_protocol._start_shutdown()
-
- def __del__(self, _warn=warnings.warn):
if not self._closed:
- _warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
- self.close()
+ self._closed = True
+ self._ssl_protocol._start_shutdown()
+ else:
+ self._ssl_protocol = None
+
+ def __del__(self, _warnings=warnings):
+ if not self._closed:
+ self._closed = True
+ _warnings.warn(
+ "unclosed transport <asyncio._SSLProtocolTransport "
+ "object>", ResourceWarning)
def is_reading(self):
- tr = self._ssl_protocol._transport
- if tr is None:
- raise RuntimeError('SSL transport has not been initialized yet')
- return tr.is_reading()
+ return not self._ssl_protocol._app_reading_paused
def pause_reading(self):
"""Pause the receiving end.
@@ -332,7 +129,7 @@
No data will be passed to the protocol's data_received()
method until resume_reading() is called.
"""
- self._ssl_protocol._transport.pause_reading()
+ self._ssl_protocol._pause_reading()
def resume_reading(self):
"""Resume the receiving end.
@@ -340,7 +137,7 @@
Data received will once again be passed to the protocol's
data_received() method.
"""
- self._ssl_protocol._transport.resume_reading()
+ self._ssl_protocol._resume_reading()
def set_write_buffer_limits(self, high=None, low=None):
"""Set the high- and low-water limits for write flow control.
@@ -361,22 +158,51 @@
reduces opportunities for doing I/O and computation
concurrently.
"""
- self._ssl_protocol._transport.set_write_buffer_limits(high, low)
-
- def get_write_buffer_size(self):
- """Return the current size of the write buffer."""
- return self._ssl_protocol._transport.get_write_buffer_size()
+ self._ssl_protocol._set_write_buffer_limits(high, low)
+ self._ssl_protocol._control_app_writing()
def get_write_buffer_limits(self):
- """Get the high and low watermarks for write flow control.
- Return a tuple (low, high) where low and high are
- positive number of bytes."""
- return self._ssl_protocol._transport.get_write_buffer_limits()
+ return (self._ssl_protocol._outgoing_low_water,
+ self._ssl_protocol._outgoing_high_water)
+
+ def get_write_buffer_size(self):
+ """Return the current size of the write buffers."""
+ return self._ssl_protocol._get_write_buffer_size()
+
+ def set_read_buffer_limits(self, high=None, low=None):
+ """Set the high- and low-water limits for read flow control.
+
+ These two values control when to call the upstream transport's
+ pause_reading() and resume_reading() methods. If specified,
+ the low-water limit must be less than or equal to the
+ high-water limit. Neither value can be negative.
+
+ The defaults are implementation-specific. If only the
+ high-water limit is given, the low-water limit defaults to an
+ implementation-specific value less than or equal to the
+ high-water limit. Setting high to zero forces low to zero as
+ well, and causes pause_reading() to be called whenever the
+ buffer becomes non-empty. Setting low to zero causes
+ resume_reading() to be called only once the buffer is empty.
+ Use of zero for either limit is generally sub-optimal as it
+ reduces opportunities for doing I/O and computation
+ concurrently.
+ """
+ self._ssl_protocol._set_read_buffer_limits(high, low)
+ self._ssl_protocol._control_ssl_reading()
+
+ def get_read_buffer_limits(self):
+ return (self._ssl_protocol._incoming_low_water,
+ self._ssl_protocol._incoming_high_water)
+
+ def get_read_buffer_size(self):
+ """Return the current size of the read buffer."""
+ return self._ssl_protocol._get_read_buffer_size()
@property
def _protocol_paused(self):
# Required for sendfile fallback pause_writing/resume_writing logic
- return self._ssl_protocol._transport._protocol_paused
+ return self._ssl_protocol._app_writing_paused
def write(self, data):
"""Write some data bytes to the transport.
@@ -389,7 +215,22 @@
f"got {type(data).__name__}")
if not data:
return
- self._ssl_protocol._write_appdata(data)
+ self._ssl_protocol._write_appdata((data,))
+
+ def writelines(self, list_of_data):
+ """Write a list (or any iterable) of data bytes to the transport.
+
+ The default implementation concatenates the arguments and
+ calls write() on the result.
+ """
+ self._ssl_protocol._write_appdata(list_of_data)
+
+ def write_eof(self):
+ """Close the write end after flushing buffered data.
+
+ This raises :exc:`NotImplementedError` right now.
+ """
+ raise NotImplementedError
def can_write_eof(self):
"""Return True if this transport supports write_eof(), False if not."""
@@ -402,23 +243,37 @@
The protocol's connection_lost() method will (eventually) be
called with None as its argument.
"""
- self._ssl_protocol._abort()
self._closed = True
+ if self._ssl_protocol is not None:
+ self._ssl_protocol._abort()
+
+ def _force_close(self, exc):
+ self._closed = True
+ self._ssl_protocol._abort(exc)
+
+ def _test__append_write_backlog(self, data):
+ # for test only
+ self._ssl_protocol._write_backlog.append(data)
+ self._ssl_protocol._write_buffer_size += len(data)
-class SSLProtocol(protocols.Protocol):
- """SSL protocol.
+class SSLProtocol(protocols.BufferedProtocol):
+ max_size = 256 * 1024 # Buffer size passed to read()
- Implementation of SSL on top of a socket using incoming and outgoing
- buffers which are ssl.MemoryBIO objects.
- """
+ _handshake_start_time = None
+ _handshake_timeout_handle = None
+ _shutdown_timeout_handle = None
def __init__(self, loop, app_protocol, sslcontext, waiter,
server_side=False, server_hostname=None,
call_connection_made=True,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
if ssl is None:
- raise RuntimeError('stdlib ssl module not available')
+ raise RuntimeError("stdlib ssl module not available")
+
+ self._ssl_buffer = bytearray(self.max_size)
+ self._ssl_buffer_view = memoryview(self._ssl_buffer)
if ssl_handshake_timeout is None:
ssl_handshake_timeout = constants.SSL_HANDSHAKE_TIMEOUT
@@ -426,6 +281,12 @@
raise ValueError(
f"ssl_handshake_timeout should be a positive number, "
f"got {ssl_handshake_timeout}")
+ if ssl_shutdown_timeout is None:
+ ssl_shutdown_timeout = constants.SSL_SHUTDOWN_TIMEOUT
+ elif ssl_shutdown_timeout <= 0:
+ raise ValueError(
+ f"ssl_shutdown_timeout should be a positive number, "
+ f"got {ssl_shutdown_timeout}")
if not sslcontext:
sslcontext = _create_transport_context(
@@ -448,21 +309,54 @@
self._waiter = waiter
self._loop = loop
self._set_app_protocol(app_protocol)
- self._app_transport = _SSLProtocolTransport(self._loop, self)
- # _SSLPipe instance (None until the connection is made)
- self._sslpipe = None
- self._session_established = False
- self._in_handshake = False
- self._in_shutdown = False
+ self._app_transport = None
+ self._app_transport_created = False
# transport, ex: SelectorSocketTransport
self._transport = None
- self._call_connection_made = call_connection_made
self._ssl_handshake_timeout = ssl_handshake_timeout
+ self._ssl_shutdown_timeout = ssl_shutdown_timeout
+ # SSL and state machine
+ self._incoming = ssl.MemoryBIO()
+ self._outgoing = ssl.MemoryBIO()
+ self._state = SSLProtocolState.UNWRAPPED
+ self._conn_lost = 0 # Set when connection_lost called
+ if call_connection_made:
+ self._app_state = AppProtocolState.STATE_INIT
+ else:
+ self._app_state = AppProtocolState.STATE_CON_MADE
+ self._sslobj = self._sslcontext.wrap_bio(
+ self._incoming, self._outgoing,
+ server_side=self._server_side,
+ server_hostname=self._server_hostname)
+
+ # Flow Control
+
+ self._ssl_writing_paused = False
+
+ self._app_reading_paused = False
+
+ self._ssl_reading_paused = False
+ self._incoming_high_water = 0
+ self._incoming_low_water = 0
+ self._set_read_buffer_limits()
+ self._eof_received = False
+
+ self._app_writing_paused = False
+ self._outgoing_high_water = 0
+ self._outgoing_low_water = 0
+ self._set_write_buffer_limits()
+ self._get_app_transport()
def _set_app_protocol(self, app_protocol):
self._app_protocol = app_protocol
- self._app_protocol_is_buffer = \
- isinstance(app_protocol, protocols.BufferedProtocol)
+ # Make fast hasattr check first
+ if (hasattr(app_protocol, 'get_buffer') and
+ isinstance(app_protocol, protocols.BufferedProtocol)):
+ self._app_protocol_get_buffer = app_protocol.get_buffer
+ self._app_protocol_buffer_updated = app_protocol.buffer_updated
+ self._app_protocol_is_buffer = True
+ else:
+ self._app_protocol_is_buffer = False
def _wakeup_waiter(self, exc=None):
if self._waiter is None:
@@ -474,15 +368,20 @@
self._waiter.set_result(None)
self._waiter = None
+ def _get_app_transport(self):
+ if self._app_transport is None:
+ if self._app_transport_created:
+ raise RuntimeError('Creating _SSLProtocolTransport twice')
+ self._app_transport = _SSLProtocolTransport(self._loop, self)
+ self._app_transport_created = True
+ return self._app_transport
+
def connection_made(self, transport):
"""Called when the low-level connection is made.
Start the SSL handshake.
"""
self._transport = transport
- self._sslpipe = _SSLPipe(self._sslcontext,
- self._server_side,
- self._server_hostname)
self._start_handshake()
def connection_lost(self, exc):
@@ -492,72 +391,58 @@
meaning a regular EOF is received or the connection was
aborted or closed).
"""
- if self._session_established:
- self._session_established = False
- self._loop.call_soon(self._app_protocol.connection_lost, exc)
- else:
- # Most likely an exception occurred while in SSL handshake.
- # Just mark the app transport as closed so that its __del__
- # doesn't complain.
- if self._app_transport is not None:
- self._app_transport._closed = True
+ self._write_backlog.clear()
+ self._outgoing.read()
+ self._conn_lost += 1
+
+ # Just mark the app transport as closed so that its __dealloc__
+ # doesn't complain.
+ if self._app_transport is not None:
+ self._app_transport._closed = True
+
+ if self._state != SSLProtocolState.DO_HANDSHAKE:
+ if (
+ self._app_state == AppProtocolState.STATE_CON_MADE or
+ self._app_state == AppProtocolState.STATE_EOF
+ ):
+ self._app_state = AppProtocolState.STATE_CON_LOST
+ self._loop.call_soon(self._app_protocol.connection_lost, exc)
+ self._set_state(SSLProtocolState.UNWRAPPED)
self._transport = None
self._app_transport = None
- if getattr(self, '_handshake_timeout_handle', None):
- self._handshake_timeout_handle.cancel()
- self._wakeup_waiter(exc)
self._app_protocol = None
- self._sslpipe = None
+ self._wakeup_waiter(exc)
- def pause_writing(self):
- """Called when the low-level transport's buffer goes over
- the high-water mark.
- """
- self._app_protocol.pause_writing()
+ if self._shutdown_timeout_handle:
+ self._shutdown_timeout_handle.cancel()
+ self._shutdown_timeout_handle = None
+ if self._handshake_timeout_handle:
+ self._handshake_timeout_handle.cancel()
+ self._handshake_timeout_handle = None
- def resume_writing(self):
- """Called when the low-level transport's buffer drains below
- the low-water mark.
- """
- self._app_protocol.resume_writing()
+ def get_buffer(self, n):
+ want = n
+ if want <= 0 or want > self.max_size:
+ want = self.max_size
+ if len(self._ssl_buffer) < want:
+ self._ssl_buffer = bytearray(want)
+ self._ssl_buffer_view = memoryview(self._ssl_buffer)
+ return self._ssl_buffer_view
- def data_received(self, data):
- """Called when some SSL data is received.
+ def buffer_updated(self, nbytes):
+ self._incoming.write(self._ssl_buffer_view[:nbytes])
- The argument is a bytes object.
- """
- if self._sslpipe is None:
- # transport closing, sslpipe is destroyed
- return
+ if self._state == SSLProtocolState.DO_HANDSHAKE:
+ self._do_handshake()
- try:
- ssldata, appdata = self._sslpipe.feed_ssldata(data)
- except (SystemExit, KeyboardInterrupt):
- raise
- except BaseException as e:
- self._fatal_error(e, 'SSL error in data received')
- return
+ elif self._state == SSLProtocolState.WRAPPED:
+ self._do_read()
- for chunk in ssldata:
- self._transport.write(chunk)
+ elif self._state == SSLProtocolState.FLUSHING:
+ self._do_flush()
- for chunk in appdata:
- if chunk:
- try:
- if self._app_protocol_is_buffer:
- protocols._feed_data_to_buffered_proto(
- self._app_protocol, chunk)
- else:
- self._app_protocol.data_received(chunk)
- except (SystemExit, KeyboardInterrupt):
- raise
- except BaseException as ex:
- self._fatal_error(
- ex, 'application protocol failed to receive SSL data')
- return
- else:
- self._start_shutdown()
- break
+ elif self._state == SSLProtocolState.SHUTDOWN:
+ self._do_shutdown()
def eof_received(self):
"""Called when the other end of the low-level stream
@@ -567,19 +452,32 @@
will close itself. If it returns a true value, closing the
transport is up to the protocol.
"""
+ self._eof_received = True
try:
if self._loop.get_debug():
logger.debug("%r received EOF", self)
- self._wakeup_waiter(ConnectionResetError)
+ if self._state == SSLProtocolState.DO_HANDSHAKE:
+ self._on_handshake_complete(ConnectionResetError)
- if not self._in_handshake:
- keep_open = self._app_protocol.eof_received()
- if keep_open:
- logger.warning('returning true from eof_received() '
- 'has no effect when using ssl')
- finally:
+ elif self._state == SSLProtocolState.WRAPPED:
+ self._set_state(SSLProtocolState.FLUSHING)
+ if self._app_reading_paused:
+ return True
+ else:
+ self._do_flush()
+
+ elif self._state == SSLProtocolState.FLUSHING:
+ self._do_write()
+ self._set_state(SSLProtocolState.SHUTDOWN)
+ self._do_shutdown()
+
+ elif self._state == SSLProtocolState.SHUTDOWN:
+ self._do_shutdown()
+
+ except Exception:
self._transport.close()
+ raise
def _get_extra_info(self, name, default=None):
if name in self._extra:
@@ -589,19 +487,45 @@
else:
return default
- def _start_shutdown(self):
- if self._in_shutdown:
- return
- if self._in_handshake:
- self._abort()
- else:
- self._in_shutdown = True
- self._write_appdata(b'')
+ def _set_state(self, new_state):
+ allowed = False
- def _write_appdata(self, data):
- self._write_backlog.append((data, 0))
- self._write_buffer_size += len(data)
- self._process_write_backlog()
+ if new_state == SSLProtocolState.UNWRAPPED:
+ allowed = True
+
+ elif (
+ self._state == SSLProtocolState.UNWRAPPED and
+ new_state == SSLProtocolState.DO_HANDSHAKE
+ ):
+ allowed = True
+
+ elif (
+ self._state == SSLProtocolState.DO_HANDSHAKE and
+ new_state == SSLProtocolState.WRAPPED
+ ):
+ allowed = True
+
+ elif (
+ self._state == SSLProtocolState.WRAPPED and
+ new_state == SSLProtocolState.FLUSHING
+ ):
+ allowed = True
+
+ elif (
+ self._state == SSLProtocolState.FLUSHING and
+ new_state == SSLProtocolState.SHUTDOWN
+ ):
+ allowed = True
+
+ if allowed:
+ self._state = new_state
+
+ else:
+ raise RuntimeError(
+ 'cannot switch state from {} to {}'.format(
+ self._state, new_state))
+
+ # Handshake flow
def _start_handshake(self):
if self._loop.get_debug():
@@ -609,17 +533,18 @@
self._handshake_start_time = self._loop.time()
else:
self._handshake_start_time = None
- self._in_handshake = True
- # (b'', 1) is a special value in _process_write_backlog() to do
- # the SSL handshake
- self._write_backlog.append((b'', 1))
+
+ self._set_state(SSLProtocolState.DO_HANDSHAKE)
+
+ # start handshake timeout count down
self._handshake_timeout_handle = \
self._loop.call_later(self._ssl_handshake_timeout,
- self._check_handshake_timeout)
- self._process_write_backlog()
+ lambda: self._check_handshake_timeout())
+
+ self._do_handshake()
def _check_handshake_timeout(self):
- if self._in_handshake is True:
+ if self._state == SSLProtocolState.DO_HANDSHAKE:
msg = (
f"SSL handshake is taking longer than "
f"{self._ssl_handshake_timeout} seconds: "
@@ -627,24 +552,37 @@
)
self._fatal_error(ConnectionAbortedError(msg))
- def _on_handshake_complete(self, handshake_exc):
- self._in_handshake = False
- self._handshake_timeout_handle.cancel()
-
- sslobj = self._sslpipe.ssl_object
+ def _do_handshake(self):
try:
- if handshake_exc is not None:
+ self._sslobj.do_handshake()
+ except SSLAgainErrors:
+ self._process_outgoing()
+ except ssl.SSLError as exc:
+ self._on_handshake_complete(exc)
+ else:
+ self._on_handshake_complete(None)
+
+ def _on_handshake_complete(self, handshake_exc):
+ if self._handshake_timeout_handle is not None:
+ self._handshake_timeout_handle.cancel()
+ self._handshake_timeout_handle = None
+
+ sslobj = self._sslobj
+ try:
+ if handshake_exc is None:
+ self._set_state(SSLProtocolState.WRAPPED)
+ else:
raise handshake_exc
peercert = sslobj.getpeercert()
- except (SystemExit, KeyboardInterrupt):
- raise
- except BaseException as exc:
+ except Exception as exc:
+ self._set_state(SSLProtocolState.UNWRAPPED)
if isinstance(exc, ssl.CertificateError):
msg = 'SSL handshake failed on verifying the certificate'
else:
msg = 'SSL handshake failed'
self._fatal_error(exc, msg)
+ self._wakeup_waiter(exc)
return
if self._loop.get_debug():
@@ -655,85 +593,330 @@
self._extra.update(peercert=peercert,
cipher=sslobj.cipher(),
compression=sslobj.compression(),
- ssl_object=sslobj,
- )
- if self._call_connection_made:
- self._app_protocol.connection_made(self._app_transport)
+ ssl_object=sslobj)
+ if self._app_state == AppProtocolState.STATE_INIT:
+ self._app_state = AppProtocolState.STATE_CON_MADE
+ self._app_protocol.connection_made(self._get_app_transport())
self._wakeup_waiter()
- self._session_established = True
- # In case transport.write() was already called. Don't call
- # immediately _process_write_backlog(), but schedule it:
- # _on_handshake_complete() can be called indirectly from
- # _process_write_backlog(), and _process_write_backlog() is not
- # reentrant.
- self._loop.call_soon(self._process_write_backlog)
+ self._do_read()
- def _process_write_backlog(self):
- # Try to make progress on the write backlog.
- if self._transport is None or self._sslpipe is None:
+ # Shutdown flow
+
+ def _start_shutdown(self):
+ if (
+ self._state in (
+ SSLProtocolState.FLUSHING,
+ SSLProtocolState.SHUTDOWN,
+ SSLProtocolState.UNWRAPPED
+ )
+ ):
+ return
+ if self._app_transport is not None:
+ self._app_transport._closed = True
+ if self._state == SSLProtocolState.DO_HANDSHAKE:
+ self._abort()
+ else:
+ self._set_state(SSLProtocolState.FLUSHING)
+ self._shutdown_timeout_handle = self._loop.call_later(
+ self._ssl_shutdown_timeout,
+ lambda: self._check_shutdown_timeout()
+ )
+ self._do_flush()
+
+ def _check_shutdown_timeout(self):
+ if (
+ self._state in (
+ SSLProtocolState.FLUSHING,
+ SSLProtocolState.SHUTDOWN
+ )
+ ):
+ self._transport._force_close(
+ exceptions.TimeoutError('SSL shutdown timed out'))
+
+ def _do_flush(self):
+ self._do_read()
+ self._set_state(SSLProtocolState.SHUTDOWN)
+ self._do_shutdown()
+
+ def _do_shutdown(self):
+ try:
+ if not self._eof_received:
+ self._sslobj.unwrap()
+ except SSLAgainErrors:
+ self._process_outgoing()
+ except ssl.SSLError as exc:
+ self._on_shutdown_complete(exc)
+ else:
+ self._process_outgoing()
+ self._call_eof_received()
+ self._on_shutdown_complete(None)
+
+ def _on_shutdown_complete(self, shutdown_exc):
+ if self._shutdown_timeout_handle is not None:
+ self._shutdown_timeout_handle.cancel()
+ self._shutdown_timeout_handle = None
+
+ if shutdown_exc:
+ self._fatal_error(shutdown_exc)
+ else:
+ self._loop.call_soon(self._transport.close)
+
+ def _abort(self):
+ self._set_state(SSLProtocolState.UNWRAPPED)
+ if self._transport is not None:
+ self._transport.abort()
+
+ # Outgoing flow
+
+ def _write_appdata(self, list_of_data):
+ if (
+ self._state in (
+ SSLProtocolState.FLUSHING,
+ SSLProtocolState.SHUTDOWN,
+ SSLProtocolState.UNWRAPPED
+ )
+ ):
+ if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+ logger.warning('SSL connection is closed')
+ self._conn_lost += 1
return
+ for data in list_of_data:
+ self._write_backlog.append(data)
+ self._write_buffer_size += len(data)
+
try:
- for i in range(len(self._write_backlog)):
- data, offset = self._write_backlog[0]
- if data:
- ssldata, offset = self._sslpipe.feed_appdata(data, offset)
- elif offset:
- ssldata = self._sslpipe.do_handshake(
- self._on_handshake_complete)
- offset = 1
+ if self._state == SSLProtocolState.WRAPPED:
+ self._do_write()
+
+ except Exception as ex:
+ self._fatal_error(ex, 'Fatal error on SSL protocol')
+
+ def _do_write(self):
+ try:
+ while self._write_backlog:
+ data = self._write_backlog[0]
+ count = self._sslobj.write(data)
+ data_len = len(data)
+ if count < data_len:
+ self._write_backlog[0] = data[count:]
+ self._write_buffer_size -= count
else:
- ssldata = self._sslpipe.shutdown(self._finalize)
- offset = 1
+ del self._write_backlog[0]
+ self._write_buffer_size -= data_len
+ except SSLAgainErrors:
+ pass
+ self._process_outgoing()
- for chunk in ssldata:
- self._transport.write(chunk)
+ def _process_outgoing(self):
+ if not self._ssl_writing_paused:
+ data = self._outgoing.read()
+ if len(data):
+ self._transport.write(data)
+ self._control_app_writing()
- if offset < len(data):
- self._write_backlog[0] = (data, offset)
- # A short write means that a write is blocked on a read
- # We need to enable reading if it is paused!
- assert self._sslpipe.need_ssldata
- if self._transport._paused:
- self._transport.resume_reading()
+ # Incoming flow
+
+ def _do_read(self):
+ if (
+ self._state not in (
+ SSLProtocolState.WRAPPED,
+ SSLProtocolState.FLUSHING,
+ )
+ ):
+ return
+ try:
+ if not self._app_reading_paused:
+ if self._app_protocol_is_buffer:
+ self._do_read__buffered()
+ else:
+ self._do_read__copied()
+ if self._write_backlog:
+ self._do_write()
+ else:
+ self._process_outgoing()
+ self._control_ssl_reading()
+ except Exception as ex:
+ self._fatal_error(ex, 'Fatal error on SSL protocol')
+
+ def _do_read__buffered(self):
+ offset = 0
+ count = 1
+
+ buf = self._app_protocol_get_buffer(self._get_read_buffer_size())
+ wants = len(buf)
+
+ try:
+ count = self._sslobj.read(wants, buf)
+
+ if count > 0:
+ offset = count
+ while offset < wants:
+ count = self._sslobj.read(wants - offset, buf[offset:])
+ if count > 0:
+ offset += count
+ else:
+ break
+ else:
+ self._loop.call_soon(lambda: self._do_read())
+ except SSLAgainErrors:
+ pass
+ if offset > 0:
+ self._app_protocol_buffer_updated(offset)
+ if not count:
+ # close_notify
+ self._call_eof_received()
+ self._start_shutdown()
+
+ def _do_read__copied(self):
+ chunk = b'1'
+ zero = True
+ one = False
+
+ try:
+ while True:
+ chunk = self._sslobj.read(self.max_size)
+ if not chunk:
break
+ if zero:
+ zero = False
+ one = True
+ first = chunk
+ elif one:
+ one = False
+ data = [first, chunk]
+ else:
+ data.append(chunk)
+ except SSLAgainErrors:
+ pass
+ if one:
+ self._app_protocol.data_received(first)
+ elif not zero:
+ self._app_protocol.data_received(b''.join(data))
+ if not chunk:
+ # close_notify
+ self._call_eof_received()
+ self._start_shutdown()
- # An entire chunk from the backlog was processed. We can
- # delete it and reduce the outstanding buffer size.
- del self._write_backlog[0]
- self._write_buffer_size -= len(data)
- except (SystemExit, KeyboardInterrupt):
+ def _call_eof_received(self):
+ try:
+ if self._app_state == AppProtocolState.STATE_CON_MADE:
+ self._app_state = AppProtocolState.STATE_EOF
+ keep_open = self._app_protocol.eof_received()
+ if keep_open:
+ logger.warning('returning true from eof_received() '
+ 'has no effect when using ssl')
+ except (KeyboardInterrupt, SystemExit):
raise
- except BaseException as exc:
- if self._in_handshake:
- # Exceptions will be re-raised in _on_handshake_complete.
- self._on_handshake_complete(exc)
- else:
- self._fatal_error(exc, 'Fatal error on SSL transport')
+ except BaseException as ex:
+ self._fatal_error(ex, 'Error calling eof_received()')
+
+ # Flow control for writes from APP socket
+
+ def _control_app_writing(self):
+ size = self._get_write_buffer_size()
+ if size >= self._outgoing_high_water and not self._app_writing_paused:
+ self._app_writing_paused = True
+ try:
+ self._app_protocol.pause_writing()
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except BaseException as exc:
+ self._loop.call_exception_handler({
+ 'message': 'protocol.pause_writing() failed',
+ 'exception': exc,
+ 'transport': self._app_transport,
+ 'protocol': self,
+ })
+ elif size <= self._outgoing_low_water and self._app_writing_paused:
+ self._app_writing_paused = False
+ try:
+ self._app_protocol.resume_writing()
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except BaseException as exc:
+ self._loop.call_exception_handler({
+ 'message': 'protocol.resume_writing() failed',
+ 'exception': exc,
+ 'transport': self._app_transport,
+ 'protocol': self,
+ })
+
+ def _get_write_buffer_size(self):
+ return self._outgoing.pending + self._write_buffer_size
+
+ def _set_write_buffer_limits(self, high=None, low=None):
+ high, low = add_flowcontrol_defaults(
+ high, low, constants.FLOW_CONTROL_HIGH_WATER_SSL_WRITE)
+ self._outgoing_high_water = high
+ self._outgoing_low_water = low
+
+ # Flow control for reads to APP socket
+
+ def _pause_reading(self):
+ self._app_reading_paused = True
+
+ def _resume_reading(self):
+ if self._app_reading_paused:
+ self._app_reading_paused = False
+
+ def resume():
+ if self._state == SSLProtocolState.WRAPPED:
+ self._do_read()
+ elif self._state == SSLProtocolState.FLUSHING:
+ self._do_flush()
+ elif self._state == SSLProtocolState.SHUTDOWN:
+ self._do_shutdown()
+ self._loop.call_soon(resume)
+
+ # Flow control for reads from SSL socket
+
+ def _control_ssl_reading(self):
+ size = self._get_read_buffer_size()
+ if size >= self._incoming_high_water and not self._ssl_reading_paused:
+ self._ssl_reading_paused = True
+ self._transport.pause_reading()
+ elif size <= self._incoming_low_water and self._ssl_reading_paused:
+ self._ssl_reading_paused = False
+ self._transport.resume_reading()
+
+ def _set_read_buffer_limits(self, high=None, low=None):
+ high, low = add_flowcontrol_defaults(
+ high, low, constants.FLOW_CONTROL_HIGH_WATER_SSL_READ)
+ self._incoming_high_water = high
+ self._incoming_low_water = low
+
+ def _get_read_buffer_size(self):
+ return self._incoming.pending
+
+ # Flow control for writes to SSL socket
+
+ def pause_writing(self):
+ """Called when the low-level transport's buffer goes over
+ the high-water mark.
+ """
+ assert not self._ssl_writing_paused
+ self._ssl_writing_paused = True
+
+ def resume_writing(self):
+ """Called when the low-level transport's buffer drains below
+ the low-water mark.
+ """
+ assert self._ssl_writing_paused
+ self._ssl_writing_paused = False
+ self._process_outgoing()
def _fatal_error(self, exc, message='Fatal error on transport'):
+ if self._transport:
+ self._transport._force_close(exc)
+
if isinstance(exc, OSError):
if self._loop.get_debug():
logger.debug("%r: %s", self, message, exc_info=True)
- else:
+ elif not isinstance(exc, exceptions.CancelledError):
self._loop.call_exception_handler({
'message': message,
'exception': exc,
'transport': self._transport,
'protocol': self,
})
- if self._transport:
- self._transport._force_close(exc)
-
- def _finalize(self):
- self._sslpipe = None
-
- if self._transport is not None:
- self._transport.close()
-
- def _abort(self):
- try:
- if self._transport is not None:
- self._transport.abort()
- finally:
- self._finalize()
diff --git a/common/py3-stdlib/asyncio/streams.py b/common/py3-stdlib/asyncio/streams.py
index 37fa7dc..560ad8a 100644
--- a/common/py3-stdlib/asyncio/streams.py
+++ b/common/py3-stdlib/asyncio/streams.py
@@ -215,6 +215,13 @@
return None
return self._stream_reader_wr()
+ def _replace_writer(self, writer):
+ loop = self._loop
+ transport = writer.transport
+ self._stream_writer = writer
+ self._transport = transport
+ self._over_ssl = transport.get_extra_info('sslcontext') is not None
+
def connection_made(self, transport):
if self._reject_connection:
context = {
@@ -370,6 +377,20 @@
await sleep(0)
await self._protocol._drain_helper()
+ async def start_tls(self, sslcontext, *,
+ server_hostname=None,
+ ssl_handshake_timeout=None):
+ """Upgrade an existing stream-based connection to TLS."""
+ server_side = self._protocol._client_connected_cb is not None
+ protocol = self._protocol
+ await self.drain()
+ new_transport = await self._loop.start_tls( # type: ignore
+ self._transport, protocol, sslcontext,
+ server_side=server_side, server_hostname=server_hostname,
+ ssl_handshake_timeout=ssl_handshake_timeout)
+ self._transport = new_transport
+ protocol._replace_writer(self)
+
class StreamReader:
diff --git a/common/py3-stdlib/asyncio/subprocess.py b/common/py3-stdlib/asyncio/subprocess.py
index cd10231..c380bbb 100644
--- a/common/py3-stdlib/asyncio/subprocess.py
+++ b/common/py3-stdlib/asyncio/subprocess.py
@@ -81,6 +81,9 @@
self._stdin_closed.set_result(None)
else:
self._stdin_closed.set_exception(exc)
+ # Since calling `wait_closed()` is not mandatory,
+ # we shouldn't log the traceback if this is not awaited.
+ self._stdin_closed._log_traceback = False
return
if fd == 1:
reader = self.stdout
diff --git a/common/py3-stdlib/asyncio/taskgroups.py b/common/py3-stdlib/asyncio/taskgroups.py
new file mode 100644
index 0000000..0fdea36
--- /dev/null
+++ b/common/py3-stdlib/asyncio/taskgroups.py
@@ -0,0 +1,236 @@
+# Adapted with permission from the EdgeDB project;
+# license: PSFL.
+
+
+__all__ = ["TaskGroup"]
+
+from . import events
+from . import exceptions
+from . import tasks
+
+
+class TaskGroup:
+ """Asynchronous context manager for managing groups of tasks.
+
+ Example use:
+
+ async with asyncio.TaskGroup() as group:
+ task1 = group.create_task(some_coroutine(...))
+ task2 = group.create_task(other_coroutine(...))
+ print("Both tasks have completed now.")
+
+ All tasks are awaited when the context manager exits.
+
+ Any exceptions other than `asyncio.CancelledError` raised within
+ a task will cancel all remaining tasks and wait for them to exit.
+ The exceptions are then combined and raised as an `ExceptionGroup`.
+ """
+ def __init__(self):
+ self._entered = False
+ self._exiting = False
+ self._aborting = False
+ self._loop = None
+ self._parent_task = None
+ self._parent_cancel_requested = False
+ self._tasks = set()
+ self._errors = []
+ self._base_error = None
+ self._on_completed_fut = None
+
+ def __repr__(self):
+ info = ['']
+ if self._tasks:
+ info.append(f'tasks={len(self._tasks)}')
+ if self._errors:
+ info.append(f'errors={len(self._errors)}')
+ if self._aborting:
+ info.append('cancelling')
+ elif self._entered:
+ info.append('entered')
+
+ info_str = ' '.join(info)
+ return f'<TaskGroup{info_str}>'
+
+ async def __aenter__(self):
+ if self._entered:
+ raise RuntimeError(
+ f"TaskGroup {self!r} has been already entered")
+ self._entered = True
+
+ if self._loop is None:
+ self._loop = events.get_running_loop()
+
+ self._parent_task = tasks.current_task(self._loop)
+ if self._parent_task is None:
+ raise RuntimeError(
+ f'TaskGroup {self!r} cannot determine the parent task')
+
+ return self
+
+ async def __aexit__(self, et, exc, tb):
+ self._exiting = True
+
+ if (exc is not None and
+ self._is_base_error(exc) and
+ self._base_error is None):
+ self._base_error = exc
+
+ propagate_cancellation_error = \
+ exc if et is exceptions.CancelledError else None
+ if self._parent_cancel_requested:
+ # If this flag is set we *must* call uncancel().
+ if self._parent_task.uncancel() == 0:
+ # If there are no pending cancellations left,
+ # don't propagate CancelledError.
+ propagate_cancellation_error = None
+
+ if et is not None:
+ if not self._aborting:
+ # Our parent task is being cancelled:
+ #
+ # async with TaskGroup() as g:
+ # g.create_task(...)
+ # await ... # <- CancelledError
+ #
+ # or there's an exception in "async with":
+ #
+ # async with TaskGroup() as g:
+ # g.create_task(...)
+ # 1 / 0
+ #
+ self._abort()
+
+ # We use while-loop here because "self._on_completed_fut"
+ # can be cancelled multiple times if our parent task
+ # is being cancelled repeatedly (or even once, when
+ # our own cancellation is already in progress)
+ while self._tasks:
+ if self._on_completed_fut is None:
+ self._on_completed_fut = self._loop.create_future()
+
+ try:
+ await self._on_completed_fut
+ except exceptions.CancelledError as ex:
+ if not self._aborting:
+ # Our parent task is being cancelled:
+ #
+ # async def wrapper():
+ # async with TaskGroup() as g:
+ # g.create_task(foo)
+ #
+ # "wrapper" is being cancelled while "foo" is
+ # still running.
+ propagate_cancellation_error = ex
+ self._abort()
+
+ self._on_completed_fut = None
+
+ assert not self._tasks
+
+ if self._base_error is not None:
+ raise self._base_error
+
+ # Propagate CancelledError if there is one, except if there
+ # are other errors -- those have priority.
+ if propagate_cancellation_error and not self._errors:
+ raise propagate_cancellation_error
+
+ if et is not None and et is not exceptions.CancelledError:
+ self._errors.append(exc)
+
+ if self._errors:
+ # Exceptions are heavy objects that can have object
+ # cycles (bad for GC); let's not keep a reference to
+ # a bunch of them.
+ try:
+ me = BaseExceptionGroup('unhandled errors in a TaskGroup', self._errors)
+ raise me from None
+ finally:
+ self._errors = None
+
+ def create_task(self, coro, *, name=None, context=None):
+ """Create a new task in this group and return it.
+
+ Similar to `asyncio.create_task`.
+ """
+ if not self._entered:
+ raise RuntimeError(f"TaskGroup {self!r} has not been entered")
+ if self._exiting and not self._tasks:
+ raise RuntimeError(f"TaskGroup {self!r} is finished")
+ if self._aborting:
+ raise RuntimeError(f"TaskGroup {self!r} is shutting down")
+ if context is None:
+ task = self._loop.create_task(coro)
+ else:
+ task = self._loop.create_task(coro, context=context)
+ tasks._set_task_name(task, name)
+ task.add_done_callback(self._on_task_done)
+ self._tasks.add(task)
+ return task
+
+ # Since Python 3.8 Tasks propagate all exceptions correctly,
+ # except for KeyboardInterrupt and SystemExit which are
+ # still considered special.
+
+ def _is_base_error(self, exc: BaseException) -> bool:
+ assert isinstance(exc, BaseException)
+ return isinstance(exc, (SystemExit, KeyboardInterrupt))
+
+ def _abort(self):
+ self._aborting = True
+
+ for t in self._tasks:
+ if not t.done():
+ t.cancel()
+
+ def _on_task_done(self, task):
+ self._tasks.discard(task)
+
+ if self._on_completed_fut is not None and not self._tasks:
+ if not self._on_completed_fut.done():
+ self._on_completed_fut.set_result(True)
+
+ if task.cancelled():
+ return
+
+ exc = task.exception()
+ if exc is None:
+ return
+
+ self._errors.append(exc)
+ if self._is_base_error(exc) and self._base_error is None:
+ self._base_error = exc
+
+ if self._parent_task.done():
+ # Not sure if this case is possible, but we want to handle
+ # it anyways.
+ self._loop.call_exception_handler({
+ 'message': f'Task {task!r} has errored out but its parent '
+ f'task {self._parent_task} is already completed',
+ 'exception': exc,
+ 'task': task,
+ })
+ return
+
+ if not self._aborting and not self._parent_cancel_requested:
+ # If parent task *is not* being cancelled, it means that we want
+ # to manually cancel it to abort whatever is being run right now
+ # in the TaskGroup. But we want to mark parent task as
+ # "not cancelled" later in __aexit__. Example situation that
+ # we need to handle:
+ #
+ # async def foo():
+ # try:
+ # async with TaskGroup() as g:
+ # g.create_task(crash_soon())
+ # await something # <- this needs to be canceled
+ # # by the TaskGroup, e.g.
+ # # foo() needs to be cancelled
+ # except Exception:
+ # # Ignore any exceptions raised in the TaskGroup
+ # pass
+ # await something_else # this line has to be called
+ # # after TaskGroup is finished.
+ self._abort()
+ self._parent_cancel_requested = True
+ self._parent_task.cancel()
diff --git a/common/py3-stdlib/asyncio/tasks.py b/common/py3-stdlib/asyncio/tasks.py
index f391586..3e07ce5 100644
--- a/common/py3-stdlib/asyncio/tasks.py
+++ b/common/py3-stdlib/asyncio/tasks.py
@@ -67,7 +67,10 @@
try:
set_name = task.set_name
except AttributeError:
- pass
+ warnings.warn("Task.set_name() was added in Python 3.8, "
+ "the method support will be mandatory for third-party "
+ "task implementations since 3.13.",
+ DeprecationWarning, stacklevel=3)
else:
set_name(name)
@@ -90,7 +93,7 @@
# status is still pending
_log_destroy_pending = True
- def __init__(self, coro, *, loop=None, name=None):
+ def __init__(self, coro, *, loop=None, name=None, context=None):
super().__init__(loop=loop)
if self._source_traceback:
del self._source_traceback[-1]
@@ -105,10 +108,14 @@
else:
self._name = str(name)
+ self._num_cancels_requested = 0
self._must_cancel = False
self._fut_waiter = None
self._coro = coro
- self._context = contextvars.copy_context()
+ if context is None:
+ self._context = contextvars.copy_context()
+ else:
+ self._context = context
self._loop.call_soon(self.__step, context=self._context)
_register_task(self)
@@ -126,8 +133,8 @@
__class_getitem__ = classmethod(GenericAlias)
- def _repr_info(self):
- return base_tasks._task_repr_info(self)
+ def __repr__(self):
+ return base_tasks._task_repr(self)
def get_coro(self):
return self._coro
@@ -197,10 +204,18 @@
task will be marked as cancelled when the wrapped coroutine
terminates with a CancelledError exception (even if cancel()
was not called).
+
+ This also increases the task's count of cancellation requests.
"""
self._log_traceback = False
if self.done():
return False
+ self._num_cancels_requested += 1
+ # These two lines are controversial. See discussion starting at
+ # https://github.com/python/cpython/pull/31394#issuecomment-1053545331
+ # Also remember that this is duplicated in _asynciomodule.c.
+ # if self._num_cancels_requested > 1:
+ # return False
if self._fut_waiter is not None:
if self._fut_waiter.cancel(msg=msg):
# Leave self._fut_waiter; it may be a Task that
@@ -212,6 +227,26 @@
self._cancel_message = msg
return True
+ def cancelling(self):
+ """Return the count of the task's cancellation requests.
+
+ This count is incremented when .cancel() is called
+ and may be decremented using .uncancel().
+ """
+ return self._num_cancels_requested
+
+ def uncancel(self):
+ """Decrement the task's count of cancellation requests.
+
+ This should be called by the party that called `cancel()` on the task
+ beforehand.
+
+ Returns the remaining number of cancellation requests.
+ """
+ if self._num_cancels_requested > 0:
+ self._num_cancels_requested -= 1
+ return self._num_cancels_requested
+
def __step(self, exc=None):
if self.done():
raise exceptions.InvalidStateError(
@@ -328,13 +363,18 @@
Task = _CTask = _asyncio.Task
-def create_task(coro, *, name=None):
+def create_task(coro, *, name=None, context=None):
"""Schedule the execution of a coroutine object in a spawn task.
Return a Task object.
"""
loop = events.get_running_loop()
- task = loop.create_task(coro)
+ if context is None:
+ # Use legacy API if context is not needed
+ task = loop.create_task(coro)
+ else:
+ task = loop.create_task(coro, context=context)
+
_set_task_name(task, name)
return task
@@ -347,7 +387,7 @@
async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED):
- """Wait for the Futures and coroutines given by fs to complete.
+ """Wait for the Futures or Tasks given by fs to complete.
The fs iterable must not be empty.
@@ -365,22 +405,16 @@
if futures.isfuture(fs) or coroutines.iscoroutine(fs):
raise TypeError(f"expect a list of futures, not {type(fs).__name__}")
if not fs:
- raise ValueError('Set of coroutines/Futures is empty.')
+ raise ValueError('Set of Tasks/Futures is empty.')
if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED):
raise ValueError(f'Invalid return_when value: {return_when}')
- loop = events.get_running_loop()
-
fs = set(fs)
if any(coroutines.iscoroutine(f) for f in fs):
- warnings.warn("The explicit passing of coroutine objects to "
- "asyncio.wait() is deprecated since Python 3.8, and "
- "scheduled for removal in Python 3.11.",
- DeprecationWarning, stacklevel=2)
+ raise TypeError("Passing coroutines is forbidden, use tasks explicitly.")
- fs = {ensure_future(f, loop=loop) for f in fs}
-
+ loop = events.get_running_loop()
return await _wait(fs, timeout, return_when, loop)
@@ -634,7 +668,7 @@
loop = events._get_event_loop(stacklevel=4)
try:
return loop.create_task(coro_or_future)
- except RuntimeError:
+ except RuntimeError:
if not called_wrap_awaitable:
coro_or_future.close()
raise
diff --git a/common/py3-stdlib/asyncio/timeouts.py b/common/py3-stdlib/asyncio/timeouts.py
new file mode 100644
index 0000000..029c468
--- /dev/null
+++ b/common/py3-stdlib/asyncio/timeouts.py
@@ -0,0 +1,164 @@
+import enum
+
+from types import TracebackType
+from typing import final, Optional, Type
+
+from . import events
+from . import exceptions
+from . import tasks
+
+
+__all__ = (
+ "Timeout",
+ "timeout",
+ "timeout_at",
+)
+
+
+class _State(enum.Enum):
+ CREATED = "created"
+ ENTERED = "active"
+ EXPIRING = "expiring"
+ EXPIRED = "expired"
+ EXITED = "finished"
+
+
+@final
+class Timeout:
+ """Asynchronous context manager for cancelling overdue coroutines.
+
+ Use `timeout()` or `timeout_at()` rather than instantiating this class directly.
+ """
+
+ def __init__(self, when: Optional[float]) -> None:
+ """Schedule a timeout that will trigger at a given loop time.
+
+ - If `when` is `None`, the timeout will never trigger.
+ - If `when < loop.time()`, the timeout will trigger on the next
+ iteration of the event loop.
+ """
+ self._state = _State.CREATED
+
+ self._timeout_handler: Optional[events.TimerHandle] = None
+ self._task: Optional[tasks.Task] = None
+ self._when = when
+
+ def when(self) -> Optional[float]:
+ """Return the current deadline."""
+ return self._when
+
+ def reschedule(self, when: Optional[float]) -> None:
+ """Reschedule the timeout."""
+ assert self._state is not _State.CREATED
+ if self._state is not _State.ENTERED:
+ raise RuntimeError(
+ f"Cannot change state of {self._state.value} Timeout",
+ )
+
+ self._when = when
+
+ if self._timeout_handler is not None:
+ self._timeout_handler.cancel()
+
+ if when is None:
+ self._timeout_handler = None
+ else:
+ loop = events.get_running_loop()
+ if when <= loop.time():
+ self._timeout_handler = loop.call_soon(self._on_timeout)
+ else:
+ self._timeout_handler = loop.call_at(when, self._on_timeout)
+
+ def expired(self) -> bool:
+ """Is timeout expired during execution?"""
+ return self._state in (_State.EXPIRING, _State.EXPIRED)
+
+ def __repr__(self) -> str:
+ info = ['']
+ if self._state is _State.ENTERED:
+ when = round(self._when, 3) if self._when is not None else None
+ info.append(f"when={when}")
+ info_str = ' '.join(info)
+ return f"<Timeout [{self._state.value}]{info_str}>"
+
+ async def __aenter__(self) -> "Timeout":
+ self._state = _State.ENTERED
+ self._task = tasks.current_task()
+ self._cancelling = self._task.cancelling()
+ if self._task is None:
+ raise RuntimeError("Timeout should be used inside a task")
+ self.reschedule(self._when)
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType],
+ ) -> Optional[bool]:
+ assert self._state in (_State.ENTERED, _State.EXPIRING)
+
+ if self._timeout_handler is not None:
+ self._timeout_handler.cancel()
+ self._timeout_handler = None
+
+ if self._state is _State.EXPIRING:
+ self._state = _State.EXPIRED
+
+ if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError:
+ # Since there are no new cancel requests, we're
+ # handling this.
+ raise TimeoutError from exc_val
+ elif self._state is _State.ENTERED:
+ self._state = _State.EXITED
+
+ return None
+
+ def _on_timeout(self) -> None:
+ assert self._state is _State.ENTERED
+ self._task.cancel()
+ self._state = _State.EXPIRING
+ # drop the reference early
+ self._timeout_handler = None
+
+
+def timeout(delay: Optional[float]) -> Timeout:
+ """Timeout async context manager.
+
+ Useful in cases when you want to apply timeout logic around block
+ of code or in cases when asyncio.wait_for is not suitable. For example:
+
+ >>> async with asyncio.timeout(10): # 10 seconds timeout
+ ... await long_running_task()
+
+
+ delay - value in seconds or None to disable timeout logic
+
+ long_running_task() is interrupted by raising asyncio.CancelledError,
+ the top-most affected timeout() context manager converts CancelledError
+ into TimeoutError.
+ """
+ loop = events.get_running_loop()
+ return Timeout(loop.time() + delay if delay is not None else None)
+
+
+def timeout_at(when: Optional[float]) -> Timeout:
+ """Schedule the timeout at absolute time.
+
+ Like timeout() but argument gives absolute time in the same clock system
+ as loop.time().
+
+ Please note: it is not POSIX time but a time with
+ undefined starting base, e.g. the time of the system power on.
+
+ >>> async with asyncio.timeout_at(loop.time() + 10):
+ ... await long_running_task()
+
+
+ when - a deadline when timeout occurs or None to disable timeout logic
+
+ long_running_task() is interrupted by raising asyncio.CancelledError,
+ the top-most affected timeout() context manager converts CancelledError
+ into TimeoutError.
+ """
+ return Timeout(when)
diff --git a/common/py3-stdlib/asyncio/transports.py b/common/py3-stdlib/asyncio/transports.py
index 73b1fa2..30fd41d 100644
--- a/common/py3-stdlib/asyncio/transports.py
+++ b/common/py3-stdlib/asyncio/transports.py
@@ -100,8 +100,8 @@
raise NotImplementedError
def get_write_buffer_limits(self):
- """Get the high and low watermarks for write flow control.
- Return a tuple (low, high) where low and high are
+ """Get the high and low watermarks for write flow control.
+ Return a tuple (low, high) where low and high are
positive number of bytes."""
raise NotImplementedError
diff --git a/common/py3-stdlib/asyncio/trsock.py b/common/py3-stdlib/asyncio/trsock.py
index e9ebcc3..c1f2047 100644
--- a/common/py3-stdlib/asyncio/trsock.py
+++ b/common/py3-stdlib/asyncio/trsock.py
@@ -1,5 +1,4 @@
import socket
-import warnings
class TransportSocket:
@@ -16,13 +15,6 @@
def __init__(self, sock: socket.socket):
self._sock = sock
- def _na(self, what):
- warnings.warn(
- f"Using {what} on sockets returned from get_extra_info('socket') "
- f"will be prohibited in asyncio 3.9. Please report your use case "
- f"to bugs.python.org.",
- DeprecationWarning, source=self)
-
@property
def family(self):
return self._sock.family
@@ -90,98 +82,6 @@
def getsockbyname(self):
return self._sock.getsockbyname()
- def accept(self):
- self._na('accept() method')
- return self._sock.accept()
-
- def connect(self, *args, **kwargs):
- self._na('connect() method')
- return self._sock.connect(*args, **kwargs)
-
- def connect_ex(self, *args, **kwargs):
- self._na('connect_ex() method')
- return self._sock.connect_ex(*args, **kwargs)
-
- def bind(self, *args, **kwargs):
- self._na('bind() method')
- return self._sock.bind(*args, **kwargs)
-
- def ioctl(self, *args, **kwargs):
- self._na('ioctl() method')
- return self._sock.ioctl(*args, **kwargs)
-
- def listen(self, *args, **kwargs):
- self._na('listen() method')
- return self._sock.listen(*args, **kwargs)
-
- def makefile(self):
- self._na('makefile() method')
- return self._sock.makefile()
-
- def sendfile(self, *args, **kwargs):
- self._na('sendfile() method')
- return self._sock.sendfile(*args, **kwargs)
-
- def close(self):
- self._na('close() method')
- return self._sock.close()
-
- def detach(self):
- self._na('detach() method')
- return self._sock.detach()
-
- def sendmsg_afalg(self, *args, **kwargs):
- self._na('sendmsg_afalg() method')
- return self._sock.sendmsg_afalg(*args, **kwargs)
-
- def sendmsg(self, *args, **kwargs):
- self._na('sendmsg() method')
- return self._sock.sendmsg(*args, **kwargs)
-
- def sendto(self, *args, **kwargs):
- self._na('sendto() method')
- return self._sock.sendto(*args, **kwargs)
-
- def send(self, *args, **kwargs):
- self._na('send() method')
- return self._sock.send(*args, **kwargs)
-
- def sendall(self, *args, **kwargs):
- self._na('sendall() method')
- return self._sock.sendall(*args, **kwargs)
-
- def set_inheritable(self, *args, **kwargs):
- self._na('set_inheritable() method')
- return self._sock.set_inheritable(*args, **kwargs)
-
- def share(self, process_id):
- self._na('share() method')
- return self._sock.share(process_id)
-
- def recv_into(self, *args, **kwargs):
- self._na('recv_into() method')
- return self._sock.recv_into(*args, **kwargs)
-
- def recvfrom_into(self, *args, **kwargs):
- self._na('recvfrom_into() method')
- return self._sock.recvfrom_into(*args, **kwargs)
-
- def recvmsg_into(self, *args, **kwargs):
- self._na('recvmsg_into() method')
- return self._sock.recvmsg_into(*args, **kwargs)
-
- def recvmsg(self, *args, **kwargs):
- self._na('recvmsg() method')
- return self._sock.recvmsg(*args, **kwargs)
-
- def recvfrom(self, *args, **kwargs):
- self._na('recvfrom() method')
- return self._sock.recvfrom(*args, **kwargs)
-
- def recv(self, *args, **kwargs):
- self._na('recv() method')
- return self._sock.recv(*args, **kwargs)
-
def settimeout(self, value):
if value == 0:
return
@@ -196,11 +96,3 @@
return
raise ValueError(
'setblocking(): transport sockets cannot be blocking')
-
- def __enter__(self):
- self._na('context manager protocol')
- return self._sock.__enter__()
-
- def __exit__(self, *err):
- self._na('context manager protocol')
- return self._sock.__exit__(*err)
diff --git a/common/py3-stdlib/asyncio/unix_events.py b/common/py3-stdlib/asyncio/unix_events.py
index faaca30..ac4519a 100644
--- a/common/py3-stdlib/asyncio/unix_events.py
+++ b/common/py3-stdlib/asyncio/unix_events.py
@@ -230,7 +230,8 @@
self, protocol_factory, path=None, *,
ssl=None, sock=None,
server_hostname=None,
- ssl_handshake_timeout=None):
+ ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None):
assert server_hostname is None or isinstance(server_hostname, str)
if ssl:
if server_hostname is None:
@@ -242,6 +243,9 @@
if ssl_handshake_timeout is not None:
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if ssl_shutdown_timeout is not None:
+ raise ValueError(
+ 'ssl_shutdown_timeout is only meaningful with ssl')
if path is not None:
if sock is not None:
@@ -268,13 +272,15 @@
transport, protocol = await self._create_connection_transport(
sock, protocol_factory, ssl, server_hostname,
- ssl_handshake_timeout=ssl_handshake_timeout)
+ ssl_handshake_timeout=ssl_handshake_timeout,
+ ssl_shutdown_timeout=ssl_shutdown_timeout)
return transport, protocol
async def create_unix_server(
self, protocol_factory, path=None, *,
sock=None, backlog=100, ssl=None,
ssl_handshake_timeout=None,
+ ssl_shutdown_timeout=None,
start_serving=True):
if isinstance(ssl, bool):
raise TypeError('ssl argument must be an SSLContext or None')
@@ -283,6 +289,10 @@
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
+ if ssl_shutdown_timeout is not None and not ssl:
+ raise ValueError(
+ 'ssl_shutdown_timeout is only meaningful with ssl')
+
if path is not None:
if sock is not None:
raise ValueError(
@@ -329,7 +339,8 @@
sock.setblocking(False)
server = base_events.Server(self, [sock], protocol_factory,
- ssl, backlog, ssl_handshake_timeout)
+ ssl, backlog, ssl_handshake_timeout,
+ ssl_shutdown_timeout)
if start_serving:
server._start_serving()
# Skip one loop iteration so that all 'loop.add_reader'
@@ -471,13 +482,21 @@
self._loop.call_soon(self._protocol.connection_made, self)
# only start reading when connection_made() has been called
- self._loop.call_soon(self._loop._add_reader,
+ self._loop.call_soon(self._add_reader,
self._fileno, self._read_ready)
if waiter is not None:
# only wake up the waiter when connection_made() has been called
self._loop.call_soon(futures._set_result_unless_cancelled,
waiter, None)
+ def _add_reader(self, fd, callback):
+ if not self.is_reading():
+ return
+ self._loop._add_reader(fd, callback)
+
+ def is_reading(self):
+ return not self._paused and not self._closing
+
def __repr__(self):
info = [self.__class__.__name__]
if self._pipe is None:
@@ -518,7 +537,7 @@
self._loop.call_soon(self._call_connection_lost, None)
def pause_reading(self):
- if self._closing or self._paused:
+ if not self.is_reading():
return
self._paused = True
self._loop._remove_reader(self._fileno)
diff --git a/common/py3-stdlib/asyncio/windows_events.py b/common/py3-stdlib/asyncio/windows_events.py
index 3204c7c..eb33551 100644
--- a/common/py3-stdlib/asyncio/windows_events.py
+++ b/common/py3-stdlib/asyncio/windows_events.py
@@ -33,8 +33,8 @@
)
-NULL = 0
-INFINITE = 0xffffffff
+NULL = _winapi.NULL
+INFINITE = _winapi.INFINITE
ERROR_CONNECTION_REFUSED = 1225
ERROR_CONNECTION_ABORTED = 1236
@@ -415,7 +415,7 @@
class IocpProactor:
"""Proactor implementation using IOCP."""
- def __init__(self, concurrency=0xffffffff):
+ def __init__(self, concurrency=INFINITE):
self._loop = None
self._results = []
self._iocp = _overlapped.CreateIoCompletionPort(
@@ -521,6 +521,26 @@
return self._register(ov, conn, finish_recv)
+ def recvfrom_into(self, conn, buf, flags=0):
+ self._register_with_iocp(conn)
+ ov = _overlapped.Overlapped(NULL)
+ try:
+ ov.WSARecvFromInto(conn.fileno(), buf, flags)
+ except BrokenPipeError:
+ return self._result((0, None))
+
+ def finish_recv(trans, key, ov):
+ try:
+ return ov.getresult()
+ except OSError as exc:
+ if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED,
+ _overlapped.ERROR_OPERATION_ABORTED):
+ raise ConnectionResetError(*exc.args)
+ else:
+ raise
+
+ return self._register(ov, conn, finish_recv)
+
def sendto(self, conn, buf, flags=0, addr=None):
self._register_with_iocp(conn)
ov = _overlapped.Overlapped(NULL)
@@ -850,7 +870,7 @@
return
# Cancel remaining registered operations.
- for address, (fut, ov, obj, callback) in list(self._cache.items()):
+ for fut, ov, obj, callback in list(self._cache.values()):
if fut.cancelled():
# Nothing to do with cancelled futures
pass
diff --git a/common/py3-stdlib/asyncore.py b/common/py3-stdlib/asyncore.py
index a360d40..57c8687 100644
--- a/common/py3-stdlib/asyncore.py
+++ b/common/py3-stdlib/asyncore.py
@@ -57,11 +57,9 @@
ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
errorcode
-warnings.warn(
- 'The asyncore module is deprecated and will be removed in Python 3.12. '
- 'The recommended replacement is asyncio',
- DeprecationWarning,
- stacklevel=2)
+_DEPRECATION_MSG = ('The {name} module is deprecated and will be removed in '
+ 'Python {remove}. The recommended replacement is asyncio')
+warnings._deprecated(__name__, _DEPRECATION_MSG, remove=(3, 12))
_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
diff --git a/common/py3-stdlib/base64.py b/common/py3-stdlib/base64.py
index e1256ad..7e9c2a2 100755
--- a/common/py3-stdlib/base64.py
+++ b/common/py3-stdlib/base64.py
@@ -76,15 +76,16 @@
normal base-64 alphabet nor the alternative alphabet are discarded prior
to the padding check. If validate is True, these non-alphabet characters
in the input result in a binascii.Error.
+ For more information about the strict base64 check, see:
+
+ https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64
"""
s = _bytes_from_decode_data(s)
if altchars is not None:
altchars = _bytes_from_decode_data(altchars)
assert len(altchars) == 2, repr(altchars)
s = s.translate(bytes.maketrans(altchars, b'+/'))
- if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):
- raise binascii.Error('Non-base64 digit found')
- return binascii.a2b_base64(s)
+ return binascii.a2b_base64(s, strict_mode=validate)
def standard_b64encode(s):
@@ -181,7 +182,7 @@
from_bytes = int.from_bytes
b32tab2 = _b32tab2[alphabet]
for i in range(0, len(s), 5):
- c = from_bytes(s[i: i + 5], 'big')
+ c = from_bytes(s[i: i + 5]) # big endian
encoded += (b32tab2[c >> 30] + # bits 1 - 10
b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
@@ -233,13 +234,13 @@
acc = (acc << 5) + b32rev[c]
except KeyError:
raise binascii.Error('Non-base32 digit found') from None
- decoded += acc.to_bytes(5, 'big')
+ decoded += acc.to_bytes(5) # big endian
# Process the last, partial quanta
if l % 8 or padchars not in {0, 1, 3, 4, 6}:
raise binascii.Error('Incorrect padding')
if padchars and decoded:
acc <<= 5 * padchars
- last = acc.to_bytes(5, 'big')
+ last = acc.to_bytes(5) # big endian
leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1
decoded[-5:] = last[:leftover]
return bytes(decoded)
@@ -566,15 +567,17 @@
def main():
"""Small main program"""
import sys, getopt
+ usage = """usage: %s [-h|-d|-e|-u|-t] [file|-]
+ -h: print this help message and exit
+ -d, -u: decode
+ -e: encode (default)
+ -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0]
try:
- opts, args = getopt.getopt(sys.argv[1:], 'deut')
+ opts, args = getopt.getopt(sys.argv[1:], 'hdeut')
except getopt.error as msg:
sys.stdout = sys.stderr
print(msg)
- print("""usage: %s [-d|-e|-u|-t] [file|-]
- -d, -u: decode
- -e: encode (default)
- -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
+ print(usage)
sys.exit(2)
func = encode
for o, a in opts:
@@ -582,6 +585,7 @@
if o == '-d': func = decode
if o == '-u': func = decode
if o == '-t': test(); return
+ if o == '-h': print(usage); return
if args and args[0] != '-':
with open(args[0], 'rb') as f:
func(f, sys.stdout.buffer)
diff --git a/common/py3-stdlib/bdb.py b/common/py3-stdlib/bdb.py
index 7f9b095..0f3eec6 100644
--- a/common/py3-stdlib/bdb.py
+++ b/common/py3-stdlib/bdb.py
@@ -574,6 +574,8 @@
line = linecache.getline(filename, lineno, frame.f_globals)
if line:
s += lprefix + line.strip()
+ else:
+ s += f'{lprefix}Warning: lineno is None'
return s
# The following methods can be called by clients to use
diff --git a/common/py3-stdlib/binhex.py b/common/py3-stdlib/binhex.py
deleted file mode 100644
index ace5217..0000000
--- a/common/py3-stdlib/binhex.py
+++ /dev/null
@@ -1,502 +0,0 @@
-"""Macintosh binhex compression/decompression.
-
-easy interface:
-binhex(inputfilename, outputfilename)
-hexbin(inputfilename, outputfilename)
-"""
-
-#
-# Jack Jansen, CWI, August 1995.
-#
-# The module is supposed to be as compatible as possible. Especially the
-# easy interface should work "as expected" on any platform.
-# XXXX Note: currently, textfiles appear in mac-form on all platforms.
-# We seem to lack a simple character-translate in python.
-# (we should probably use ISO-Latin-1 on all but the mac platform).
-# XXXX The simple routines are too simple: they expect to hold the complete
-# files in-core. Should be fixed.
-# XXXX It would be nice to handle AppleDouble format on unix
-# (for servers serving macs).
-# XXXX I don't understand what happens when you get 0x90 times the same byte on
-# input. The resulting code (xx 90 90) would appear to be interpreted as an
-# escaped *value* of 0x90. All coders I've seen appear to ignore this nicety...
-#
-import binascii
-import contextlib
-import io
-import os
-import struct
-import warnings
-
-warnings.warn('the binhex module is deprecated', DeprecationWarning,
- stacklevel=2)
-
-
-__all__ = ["binhex","hexbin","Error"]
-
-class Error(Exception):
- pass
-
-# States (what have we written)
-_DID_HEADER = 0
-_DID_DATA = 1
-
-# Various constants
-REASONABLY_LARGE = 32768 # Minimal amount we pass the rle-coder
-LINELEN = 64
-RUNCHAR = b"\x90"
-
-#
-# This code is no longer byte-order dependent
-
-
-class FInfo:
- def __init__(self):
- self.Type = '????'
- self.Creator = '????'
- self.Flags = 0
-
-def getfileinfo(name):
- finfo = FInfo()
- with io.open(name, 'rb') as fp:
- # Quick check for textfile
- data = fp.read(512)
- if 0 not in data:
- finfo.Type = 'TEXT'
- fp.seek(0, 2)
- dsize = fp.tell()
- dir, file = os.path.split(name)
- file = file.replace(':', '-', 1)
- return file, finfo, dsize, 0
-
-class openrsrc:
- def __init__(self, *args):
- pass
-
- def read(self, *args):
- return b''
-
- def write(self, *args):
- pass
-
- def close(self):
- pass
-
-
-# DeprecationWarning is already emitted on "import binhex". There is no need
-# to repeat the warning at each call to deprecated binascii functions.
-@contextlib.contextmanager
-def _ignore_deprecation_warning():
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', '', DeprecationWarning)
- yield
-
-
-class _Hqxcoderengine:
- """Write data to the coder in 3-byte chunks"""
-
- def __init__(self, ofp):
- self.ofp = ofp
- self.data = b''
- self.hqxdata = b''
- self.linelen = LINELEN - 1
-
- def write(self, data):
- self.data = self.data + data
- datalen = len(self.data)
- todo = (datalen // 3) * 3
- data = self.data[:todo]
- self.data = self.data[todo:]
- if not data:
- return
- with _ignore_deprecation_warning():
- self.hqxdata = self.hqxdata + binascii.b2a_hqx(data)
- self._flush(0)
-
- def _flush(self, force):
- first = 0
- while first <= len(self.hqxdata) - self.linelen:
- last = first + self.linelen
- self.ofp.write(self.hqxdata[first:last] + b'\r')
- self.linelen = LINELEN
- first = last
- self.hqxdata = self.hqxdata[first:]
- if force:
- self.ofp.write(self.hqxdata + b':\r')
-
- def close(self):
- if self.data:
- with _ignore_deprecation_warning():
- self.hqxdata = self.hqxdata + binascii.b2a_hqx(self.data)
- self._flush(1)
- self.ofp.close()
- del self.ofp
-
-class _Rlecoderengine:
- """Write data to the RLE-coder in suitably large chunks"""
-
- def __init__(self, ofp):
- self.ofp = ofp
- self.data = b''
-
- def write(self, data):
- self.data = self.data + data
- if len(self.data) < REASONABLY_LARGE:
- return
- with _ignore_deprecation_warning():
- rledata = binascii.rlecode_hqx(self.data)
- self.ofp.write(rledata)
- self.data = b''
-
- def close(self):
- if self.data:
- with _ignore_deprecation_warning():
- rledata = binascii.rlecode_hqx(self.data)
- self.ofp.write(rledata)
- self.ofp.close()
- del self.ofp
-
-class BinHex:
- def __init__(self, name_finfo_dlen_rlen, ofp):
- name, finfo, dlen, rlen = name_finfo_dlen_rlen
- close_on_error = False
- if isinstance(ofp, str):
- ofname = ofp
- ofp = io.open(ofname, 'wb')
- close_on_error = True
- try:
- ofp.write(b'(This file must be converted with BinHex 4.0)\r\r:')
- hqxer = _Hqxcoderengine(ofp)
- self.ofp = _Rlecoderengine(hqxer)
- self.crc = 0
- if finfo is None:
- finfo = FInfo()
- self.dlen = dlen
- self.rlen = rlen
- self._writeinfo(name, finfo)
- self.state = _DID_HEADER
- except:
- if close_on_error:
- ofp.close()
- raise
-
- def _writeinfo(self, name, finfo):
- nl = len(name)
- if nl > 63:
- raise Error('Filename too long')
- d = bytes([nl]) + name.encode("latin-1") + b'\0'
- tp, cr = finfo.Type, finfo.Creator
- if isinstance(tp, str):
- tp = tp.encode("latin-1")
- if isinstance(cr, str):
- cr = cr.encode("latin-1")
- d2 = tp + cr
-
- # Force all structs to be packed with big-endian
- d3 = struct.pack('>h', finfo.Flags)
- d4 = struct.pack('>ii', self.dlen, self.rlen)
- info = d + d2 + d3 + d4
- self._write(info)
- self._writecrc()
-
- def _write(self, data):
- self.crc = binascii.crc_hqx(data, self.crc)
- self.ofp.write(data)
-
- def _writecrc(self):
- # XXXX Should this be here??
- # self.crc = binascii.crc_hqx('\0\0', self.crc)
- if self.crc < 0:
- fmt = '>h'
- else:
- fmt = '>H'
- self.ofp.write(struct.pack(fmt, self.crc))
- self.crc = 0
-
- def write(self, data):
- if self.state != _DID_HEADER:
- raise Error('Writing data at the wrong time')
- self.dlen = self.dlen - len(data)
- self._write(data)
-
- def close_data(self):
- if self.dlen != 0:
- raise Error('Incorrect data size, diff=%r' % (self.rlen,))
- self._writecrc()
- self.state = _DID_DATA
-
- def write_rsrc(self, data):
- if self.state < _DID_DATA:
- self.close_data()
- if self.state != _DID_DATA:
- raise Error('Writing resource data at the wrong time')
- self.rlen = self.rlen - len(data)
- self._write(data)
-
- def close(self):
- if self.state is None:
- return
- try:
- if self.state < _DID_DATA:
- self.close_data()
- if self.state != _DID_DATA:
- raise Error('Close at the wrong time')
- if self.rlen != 0:
- raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,))
- self._writecrc()
- finally:
- self.state = None
- ofp = self.ofp
- del self.ofp
- ofp.close()
-
-def binhex(inp, out):
- """binhex(infilename, outfilename): create binhex-encoded copy of a file"""
- finfo = getfileinfo(inp)
- ofp = BinHex(finfo, out)
-
- with io.open(inp, 'rb') as ifp:
- # XXXX Do textfile translation on non-mac systems
- while True:
- d = ifp.read(128000)
- if not d: break
- ofp.write(d)
- ofp.close_data()
-
- ifp = openrsrc(inp, 'rb')
- while True:
- d = ifp.read(128000)
- if not d: break
- ofp.write_rsrc(d)
- ofp.close()
- ifp.close()
-
-class _Hqxdecoderengine:
- """Read data via the decoder in 4-byte chunks"""
-
- def __init__(self, ifp):
- self.ifp = ifp
- self.eof = 0
-
- def read(self, totalwtd):
- """Read at least wtd bytes (or until EOF)"""
- decdata = b''
- wtd = totalwtd
- #
- # The loop here is convoluted, since we don't really now how
- # much to decode: there may be newlines in the incoming data.
- while wtd > 0:
- if self.eof: return decdata
- wtd = ((wtd + 2) // 3) * 4
- data = self.ifp.read(wtd)
- #
- # Next problem: there may not be a complete number of
- # bytes in what we pass to a2b. Solve by yet another
- # loop.
- #
- while True:
- try:
- with _ignore_deprecation_warning():
- decdatacur, self.eof = binascii.a2b_hqx(data)
- break
- except binascii.Incomplete:
- pass
- newdata = self.ifp.read(1)
- if not newdata:
- raise Error('Premature EOF on binhex file')
- data = data + newdata
- decdata = decdata + decdatacur
- wtd = totalwtd - len(decdata)
- if not decdata and not self.eof:
- raise Error('Premature EOF on binhex file')
- return decdata
-
- def close(self):
- self.ifp.close()
-
-class _Rledecoderengine:
- """Read data via the RLE-coder"""
-
- def __init__(self, ifp):
- self.ifp = ifp
- self.pre_buffer = b''
- self.post_buffer = b''
- self.eof = 0
-
- def read(self, wtd):
- if wtd > len(self.post_buffer):
- self._fill(wtd - len(self.post_buffer))
- rv = self.post_buffer[:wtd]
- self.post_buffer = self.post_buffer[wtd:]
- return rv
-
- def _fill(self, wtd):
- self.pre_buffer = self.pre_buffer + self.ifp.read(wtd + 4)
- if self.ifp.eof:
- with _ignore_deprecation_warning():
- self.post_buffer = self.post_buffer + \
- binascii.rledecode_hqx(self.pre_buffer)
- self.pre_buffer = b''
- return
-
- #
- # Obfuscated code ahead. We have to take care that we don't
- # end up with an orphaned RUNCHAR later on. So, we keep a couple
- # of bytes in the buffer, depending on what the end of
- # the buffer looks like:
- # '\220\0\220' - Keep 3 bytes: repeated \220 (escaped as \220\0)
- # '?\220' - Keep 2 bytes: repeated something-else
- # '\220\0' - Escaped \220: Keep 2 bytes.
- # '?\220?' - Complete repeat sequence: decode all
- # otherwise: keep 1 byte.
- #
- mark = len(self.pre_buffer)
- if self.pre_buffer[-3:] == RUNCHAR + b'\0' + RUNCHAR:
- mark = mark - 3
- elif self.pre_buffer[-1:] == RUNCHAR:
- mark = mark - 2
- elif self.pre_buffer[-2:] == RUNCHAR + b'\0':
- mark = mark - 2
- elif self.pre_buffer[-2:-1] == RUNCHAR:
- pass # Decode all
- else:
- mark = mark - 1
-
- with _ignore_deprecation_warning():
- self.post_buffer = self.post_buffer + \
- binascii.rledecode_hqx(self.pre_buffer[:mark])
- self.pre_buffer = self.pre_buffer[mark:]
-
- def close(self):
- self.ifp.close()
-
-class HexBin:
- def __init__(self, ifp):
- if isinstance(ifp, str):
- ifp = io.open(ifp, 'rb')
- #
- # Find initial colon.
- #
- while True:
- ch = ifp.read(1)
- if not ch:
- raise Error("No binhex data found")
- # Cater for \r\n terminated lines (which show up as \n\r, hence
- # all lines start with \r)
- if ch == b'\r':
- continue
- if ch == b':':
- break
-
- hqxifp = _Hqxdecoderengine(ifp)
- self.ifp = _Rledecoderengine(hqxifp)
- self.crc = 0
- self._readheader()
-
- def _read(self, len):
- data = self.ifp.read(len)
- self.crc = binascii.crc_hqx(data, self.crc)
- return data
-
- def _checkcrc(self):
- filecrc = struct.unpack('>h', self.ifp.read(2))[0] & 0xffff
- #self.crc = binascii.crc_hqx('\0\0', self.crc)
- # XXXX Is this needed??
- self.crc = self.crc & 0xffff
- if filecrc != self.crc:
- raise Error('CRC error, computed %x, read %x'
- % (self.crc, filecrc))
- self.crc = 0
-
- def _readheader(self):
- len = self._read(1)
- fname = self._read(ord(len))
- rest = self._read(1 + 4 + 4 + 2 + 4 + 4)
- self._checkcrc()
-
- type = rest[1:5]
- creator = rest[5:9]
- flags = struct.unpack('>h', rest[9:11])[0]
- self.dlen = struct.unpack('>l', rest[11:15])[0]
- self.rlen = struct.unpack('>l', rest[15:19])[0]
-
- self.FName = fname
- self.FInfo = FInfo()
- self.FInfo.Creator = creator
- self.FInfo.Type = type
- self.FInfo.Flags = flags
-
- self.state = _DID_HEADER
-
- def read(self, *n):
- if self.state != _DID_HEADER:
- raise Error('Read data at wrong time')
- if n:
- n = n[0]
- n = min(n, self.dlen)
- else:
- n = self.dlen
- rv = b''
- while len(rv) < n:
- rv = rv + self._read(n-len(rv))
- self.dlen = self.dlen - n
- return rv
-
- def close_data(self):
- if self.state != _DID_HEADER:
- raise Error('close_data at wrong time')
- if self.dlen:
- dummy = self._read(self.dlen)
- self._checkcrc()
- self.state = _DID_DATA
-
- def read_rsrc(self, *n):
- if self.state == _DID_HEADER:
- self.close_data()
- if self.state != _DID_DATA:
- raise Error('Read resource data at wrong time')
- if n:
- n = n[0]
- n = min(n, self.rlen)
- else:
- n = self.rlen
- self.rlen = self.rlen - n
- return self._read(n)
-
- def close(self):
- if self.state is None:
- return
- try:
- if self.rlen:
- dummy = self.read_rsrc(self.rlen)
- self._checkcrc()
- finally:
- self.state = None
- self.ifp.close()
-
-def hexbin(inp, out):
- """hexbin(infilename, outfilename) - Decode binhexed file"""
- ifp = HexBin(inp)
- finfo = ifp.FInfo
- if not out:
- out = ifp.FName
-
- with io.open(out, 'wb') as ofp:
- # XXXX Do translation on non-mac systems
- while True:
- d = ifp.read(128000)
- if not d: break
- ofp.write(d)
- ifp.close_data()
-
- d = ifp.read_rsrc(128000)
- if d:
- ofp = openrsrc(out, 'wb')
- ofp.write(d)
- while True:
- d = ifp.read_rsrc(128000)
- if not d: break
- ofp.write(d)
- ofp.close()
-
- ifp.close()
diff --git a/common/py3-stdlib/cProfile.py b/common/py3-stdlib/cProfile.py
index 22a7d0a..44c028f 100755
--- a/common/py3-stdlib/cProfile.py
+++ b/common/py3-stdlib/cProfile.py
@@ -7,6 +7,7 @@
__all__ = ["run", "runctx", "Profile"]
import _lsprof
+import io
import profile as _pyprofile
# ____________________________________________________________
@@ -140,7 +141,7 @@
help="Save stats to <outfile>", default=None)
parser.add_option('-s', '--sort', dest="sort",
help="Sort order when printing to stdout, based on pstats.Stats class",
- default=-1,
+ default=2,
choices=sorted(pstats.Stats.sort_arg_dict_default))
parser.add_option('-m', dest="module", action="store_true",
help="Profile a library module", default=False)
@@ -167,7 +168,7 @@
else:
progname = args[0]
sys.path.insert(0, os.path.dirname(progname))
- with open(progname, 'rb') as fp:
+ with io.open_code(progname) as fp:
code = compile(fp.read(), progname, 'exec')
globs = {
'__file__': progname,
diff --git a/common/py3-stdlib/calendar.py b/common/py3-stdlib/calendar.py
index cbea9ec..6573964 100644
--- a/common/py3-stdlib/calendar.py
+++ b/common/py3-stdlib/calendar.py
@@ -548,27 +548,38 @@
class different_locale:
def __init__(self, locale):
self.locale = locale
+ self.oldlocale = None
def __enter__(self):
- self.oldlocale = _locale.getlocale(_locale.LC_TIME)
+ self.oldlocale = _locale.setlocale(_locale.LC_TIME, None)
_locale.setlocale(_locale.LC_TIME, self.locale)
def __exit__(self, *args):
+ if self.oldlocale is None:
+ return
_locale.setlocale(_locale.LC_TIME, self.oldlocale)
+def _get_default_locale():
+ locale = _locale.setlocale(_locale.LC_TIME, None)
+ if locale == "C":
+ with different_locale(""):
+ # The LC_TIME locale does not seem to be configured:
+ # get the user preferred locale.
+ locale = _locale.setlocale(_locale.LC_TIME, None)
+ return locale
+
+
class LocaleTextCalendar(TextCalendar):
"""
This class can be passed a locale name in the constructor and will return
- month and weekday names in the specified locale. If this locale includes
- an encoding all strings containing month and weekday names will be returned
- as unicode.
+ month and weekday names in the specified locale.
"""
def __init__(self, firstweekday=0, locale=None):
TextCalendar.__init__(self, firstweekday)
if locale is None:
- locale = _locale.getdefaultlocale()
+ locale = _get_default_locale()
self.locale = locale
def formatweekday(self, day, width):
@@ -583,14 +594,12 @@
class LocaleHTMLCalendar(HTMLCalendar):
"""
This class can be passed a locale name in the constructor and will return
- month and weekday names in the specified locale. If this locale includes
- an encoding all strings containing month and weekday names will be returned
- as unicode.
+ month and weekday names in the specified locale.
"""
def __init__(self, firstweekday=0, locale=None):
HTMLCalendar.__init__(self, firstweekday)
if locale is None:
- locale = _locale.getdefaultlocale()
+ locale = _get_default_locale()
self.locale = locale
def formatweekday(self, day):
diff --git a/common/py3-stdlib/cgi.py b/common/py3-stdlib/cgi.py
index 6cb8cf2..8787567 100755
--- a/common/py3-stdlib/cgi.py
+++ b/common/py3-stdlib/cgi.py
@@ -13,6 +13,11 @@
This module defines a number of utilities for use by CGI scripts
written in Python.
+
+The global variable maxlen can be set to an integer indicating the maximum size
+of a POST request. POST requests larger than this size will result in a
+ValueError being raised during parsing. The default value of this variable is 0,
+meaning the request size is unlimited.
"""
# History
@@ -48,6 +53,9 @@
"print_form", "print_directory", "print_arguments",
"print_environ_usage"]
+
+warnings._deprecated(__name__, remove=(3,13))
+
# Logging support
# ===============
diff --git a/common/py3-stdlib/cgitb.py b/common/py3-stdlib/cgitb.py
index 17ddda3..8ce0e83 100644
--- a/common/py3-stdlib/cgitb.py
+++ b/common/py3-stdlib/cgitb.py
@@ -31,6 +31,11 @@
import time
import tokenize
import traceback
+import warnings
+from html import escape as html_escape
+
+warnings._deprecated(__name__, remove=(3, 13))
+
def reset():
"""Return a string that resets the CGI and browser to a known state."""
@@ -105,10 +110,16 @@
etype = etype.__name__
pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
date = time.ctime(time.time())
- head = '<body bgcolor="#f0f0f8">' + pydoc.html.heading(
- '<big><big>%s</big></big>' %
- strong(pydoc.html.escape(str(etype))),
- '#ffffff', '#6622aa', pyver + '<br>' + date) + '''
+ head = f'''
+<body bgcolor="#f0f0f8">
+<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
+<tr bgcolor="#6622aa">
+<td valign=bottom> <br>
+<font color="#ffffff" face="helvetica, arial"> <br>
+<big><big><strong>{html_escape(str(etype))}</strong></big></big></font></td>
+<td align=right valign=bottom>
+<font color="#ffffff" face="helvetica, arial">{pyver}<br>{date}</font></td>
+</tr></table>
<p>A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.</p>'''
diff --git a/common/py3-stdlib/chunk.py b/common/py3-stdlib/chunk.py
index 870c39f..618781e 100644
--- a/common/py3-stdlib/chunk.py
+++ b/common/py3-stdlib/chunk.py
@@ -48,6 +48,10 @@
default is 1, i.e. aligned.
"""
+import warnings
+
+warnings._deprecated(__name__, remove=(3, 13))
+
class Chunk:
def __init__(self, file, align=True, bigendian=True, inclheader=False):
import struct
diff --git a/common/py3-stdlib/cmd.py b/common/py3-stdlib/cmd.py
index 859e910..88ee7d3 100644
--- a/common/py3-stdlib/cmd.py
+++ b/common/py3-stdlib/cmd.py
@@ -310,10 +310,10 @@
names = self.get_names()
cmds_doc = []
cmds_undoc = []
- help = {}
+ topics = set()
for name in names:
if name[:5] == 'help_':
- help[name[5:]]=1
+ topics.add(name[5:])
names.sort()
# There can be duplicates if routines overridden
prevname = ''
@@ -323,16 +323,16 @@
continue
prevname = name
cmd=name[3:]
- if cmd in help:
+ if cmd in topics:
cmds_doc.append(cmd)
- del help[cmd]
+ topics.remove(cmd)
elif getattr(self, name).__doc__:
cmds_doc.append(cmd)
else:
cmds_undoc.append(cmd)
self.stdout.write("%s\n"%str(self.doc_leader))
self.print_topics(self.doc_header, cmds_doc, 15,80)
- self.print_topics(self.misc_header, list(help.keys()),15,80)
+ self.print_topics(self.misc_header, sorted(topics),15,80)
self.print_topics(self.undoc_header, cmds_undoc, 15,80)
def print_topics(self, header, cmds, cmdlen, maxcol):
diff --git a/common/py3-stdlib/codeop.py b/common/py3-stdlib/codeop.py
index fe8713e..2213b69 100644
--- a/common/py3-stdlib/codeop.py
+++ b/common/py3-stdlib/codeop.py
@@ -43,7 +43,7 @@
# The following flags match the values from Include/cpython/compile.h
# Caveat emptor: These flags are undocumented on purpose and depending
# on their effect outside the standard library is **unsupported**.
-PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_DONT_IMPLY_DEDENT = 0x200
PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
def _maybe_compile(compiler, source, filename, symbol):
diff --git a/common/py3-stdlib/collections/__init__.py b/common/py3-stdlib/collections/__init__.py
index 818588f..7af8dcd 100644
--- a/common/py3-stdlib/collections/__init__.py
+++ b/common/py3-stdlib/collections/__init__.py
@@ -236,11 +236,19 @@
is raised.
'''
- if key in self:
- result = self[key]
- del self[key]
+ marker = self.__marker
+ result = dict.pop(self, key, marker)
+ if result is not marker:
+ # The same as in __delitem__().
+ link = self.__map.pop(key)
+ link_prev = link.prev
+ link_next = link.next
+ link_prev.next = link_next
+ link_next.prev = link_prev
+ link.prev = None
+ link.next = None
return result
- if default is self.__marker:
+ if default is marker:
raise KeyError(key)
return default
@@ -263,10 +271,22 @@
def __reduce__(self):
'Return state information for pickling'
- inst_dict = vars(self).copy()
- for k in vars(OrderedDict()):
- inst_dict.pop(k, None)
- return self.__class__, (), inst_dict or None, None, iter(self.items())
+ state = self.__getstate__()
+ if state:
+ if isinstance(state, tuple):
+ state, slots = state
+ else:
+ slots = {}
+ state = state.copy()
+ slots = slots.copy()
+ for k in vars(OrderedDict()):
+ state.pop(k, None)
+ slots.pop(k, None)
+ if slots:
+ state = state, slots
+ else:
+ state = state or None
+ return self.__class__, (), state, None, iter(self.items())
def copy(self):
'od.copy() -> a shallow copy of od'
@@ -609,11 +629,9 @@
['A', 'A', 'B', 'B', 'C', 'C']
# Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1
+ >>> import math
>>> prime_factors = Counter({2: 2, 3: 3, 17: 1})
- >>> product = 1
- >>> for factor in prime_factors.elements(): # loop over factors
- ... product *= factor # and multiply them
- >>> product
+ >>> math.prod(prime_factors.elements())
1836
Note, if an element's count has been set to zero or is a negative
@@ -710,42 +728,6 @@
if elem in self:
super().__delitem__(elem)
- def __eq__(self, other):
- 'True if all counts agree. Missing counts are treated as zero.'
- if not isinstance(other, Counter):
- return NotImplemented
- return all(self[e] == other[e] for c in (self, other) for e in c)
-
- def __ne__(self, other):
- 'True if any counts disagree. Missing counts are treated as zero.'
- if not isinstance(other, Counter):
- return NotImplemented
- return not self == other
-
- def __le__(self, other):
- 'True if all counts in self are a subset of those in other.'
- if not isinstance(other, Counter):
- return NotImplemented
- return all(self[e] <= other[e] for c in (self, other) for e in c)
-
- def __lt__(self, other):
- 'True if all counts in self are a proper subset of those in other.'
- if not isinstance(other, Counter):
- return NotImplemented
- return self <= other and self != other
-
- def __ge__(self, other):
- 'True if all counts in self are a superset of those in other.'
- if not isinstance(other, Counter):
- return NotImplemented
- return all(self[e] >= other[e] for c in (self, other) for e in c)
-
- def __gt__(self, other):
- 'True if all counts in self are a proper superset of those in other.'
- if not isinstance(other, Counter):
- return NotImplemented
- return self >= other and self != other
-
def __repr__(self):
if not self:
return f'{self.__class__.__name__}()'
@@ -791,6 +773,42 @@
# (cp >= cq) == (sp >= sq)
# (cp > cq) == (sp > sq)
+ def __eq__(self, other):
+ 'True if all counts agree. Missing counts are treated as zero.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return all(self[e] == other[e] for c in (self, other) for e in c)
+
+ def __ne__(self, other):
+ 'True if any counts disagree. Missing counts are treated as zero.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return not self == other
+
+ def __le__(self, other):
+ 'True if all counts in self are a subset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return all(self[e] <= other[e] for c in (self, other) for e in c)
+
+ def __lt__(self, other):
+ 'True if all counts in self are a proper subset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return self <= other and self != other
+
+ def __ge__(self, other):
+ 'True if all counts in self are a superset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return all(self[e] >= other[e] for c in (self, other) for e in c)
+
+ def __gt__(self, other):
+ 'True if all counts in self are a proper superset of those in other.'
+ if not isinstance(other, Counter):
+ return NotImplemented
+ return self >= other and self != other
+
def __add__(self, other):
'''Add counts from two counters.
diff --git a/common/py3-stdlib/colorsys.py b/common/py3-stdlib/colorsys.py
index 0f52512..9bdc83e 100644
--- a/common/py3-stdlib/colorsys.py
+++ b/common/py3-stdlib/colorsys.py
@@ -125,13 +125,14 @@
def rgb_to_hsv(r, g, b):
maxc = max(r, g, b)
minc = min(r, g, b)
+ rangec = (maxc-minc)
v = maxc
if minc == maxc:
return 0.0, 0.0, v
- s = (maxc-minc) / maxc
- rc = (maxc-r) / (maxc-minc)
- gc = (maxc-g) / (maxc-minc)
- bc = (maxc-b) / (maxc-minc)
+ s = rangec / maxc
+ rc = (maxc-r) / rangec
+ gc = (maxc-g) / rangec
+ bc = (maxc-b) / rangec
if r == maxc:
h = bc-gc
elif g == maxc:
diff --git a/common/py3-stdlib/compileall.py b/common/py3-stdlib/compileall.py
index 50183ea..a388931 100644
--- a/common/py3-stdlib/compileall.py
+++ b/common/py3-stdlib/compileall.py
@@ -4,7 +4,7 @@
given as arguments recursively; the -l option prevents it from
recursing into directories.
-Without arguments, if compiles all modules on sys.path, without
+Without arguments, it compiles all modules on sys.path, without
recursing into subdirectories. (Even though it should do so for
packages -- for now, you'll have to deal with packages separately.)
diff --git a/common/py3-stdlib/concurrent/futures/__init__.py b/common/py3-stdlib/concurrent/futures/__init__.py
index d746aea..292e886 100644
--- a/common/py3-stdlib/concurrent/futures/__init__.py
+++ b/common/py3-stdlib/concurrent/futures/__init__.py
@@ -50,4 +50,4 @@
ThreadPoolExecutor = te
return te
- raise AttributeError(f"module {__name__} has no attribute {name}")
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/common/py3-stdlib/concurrent/futures/_base.py b/common/py3-stdlib/concurrent/futures/_base.py
index a329e74..6742a07 100644
--- a/common/py3-stdlib/concurrent/futures/_base.py
+++ b/common/py3-stdlib/concurrent/futures/_base.py
@@ -50,9 +50,7 @@
"""The Future was cancelled."""
pass
-class TimeoutError(Error):
- """The operation exceeded the given deadline."""
- pass
+TimeoutError = TimeoutError # make local alias for the standard exception
class InvalidStateError(Error):
"""The operation is not allowed in this state."""
@@ -284,7 +282,7 @@
A named 2-tuple of sets. The first set, named 'done', contains the
futures that completed (is finished or cancelled) before the wait
completed. The second set, named 'not_done', contains uncompleted
- futures. Duplicate futures given to *fs* are removed and will be
+ futures. Duplicate futures given to *fs* are removed and will be
returned only once.
"""
fs = set(fs)
diff --git a/common/py3-stdlib/concurrent/futures/process.py b/common/py3-stdlib/concurrent/futures/process.py
index 81cc1fb..392d196 100644
--- a/common/py3-stdlib/concurrent/futures/process.py
+++ b/common/py3-stdlib/concurrent/futures/process.py
@@ -56,7 +56,7 @@
from functools import partial
import itertools
import sys
-import traceback
+from traceback import format_exception
_threads_wakeups = weakref.WeakKeyDictionary()
@@ -123,8 +123,7 @@
class _ExceptionWithTraceback:
def __init__(self, exc, tb):
- tb = traceback.format_exception(type(exc), exc, tb)
- tb = ''.join(tb)
+ tb = ''.join(format_exception(type(exc), exc, tb))
self.exc = exc
# Traceback object needs to be garbage-collected as its frames
# contain references to all the objects in the exception scope
@@ -145,10 +144,11 @@
self.kwargs = kwargs
class _ResultItem(object):
- def __init__(self, work_id, exception=None, result=None):
+ def __init__(self, work_id, exception=None, result=None, exit_pid=None):
self.work_id = work_id
self.exception = exception
self.result = result
+ self.exit_pid = exit_pid
class _CallItem(object):
def __init__(self, work_id, fn, args, kwargs):
@@ -169,7 +169,7 @@
def _on_queue_feeder_error(self, e, obj):
if isinstance(obj, _CallItem):
- tb = traceback.format_exception(type(e), e, e.__traceback__)
+ tb = format_exception(type(e), e, e.__traceback__)
e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb)))
work_item = self.pending_work_items.pop(obj.work_id, None)
with self.shutdown_lock:
@@ -205,17 +205,19 @@
return [fn(*args) for args in chunk]
-def _sendback_result(result_queue, work_id, result=None, exception=None):
+def _sendback_result(result_queue, work_id, result=None, exception=None,
+ exit_pid=None):
"""Safely send back the given result or exception"""
try:
result_queue.put(_ResultItem(work_id, result=result,
- exception=exception))
+ exception=exception, exit_pid=exit_pid))
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
- result_queue.put(_ResultItem(work_id, exception=exc))
+ result_queue.put(_ResultItem(work_id, exception=exc,
+ exit_pid=exit_pid))
-def _process_worker(call_queue, result_queue, initializer, initargs):
+def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None):
"""Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
@@ -236,25 +238,38 @@
# The parent will notice that the process stopped and
# mark the pool broken
return
+ num_tasks = 0
+ exit_pid = None
while True:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(os.getpid())
return
+
+ if max_tasks is not None:
+ num_tasks += 1
+ if num_tasks >= max_tasks:
+ exit_pid = os.getpid()
+
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException as e:
exc = _ExceptionWithTraceback(e, e.__traceback__)
- _sendback_result(result_queue, call_item.work_id, exception=exc)
+ _sendback_result(result_queue, call_item.work_id, exception=exc,
+ exit_pid=exit_pid)
else:
- _sendback_result(result_queue, call_item.work_id, result=r)
+ _sendback_result(result_queue, call_item.work_id, result=r,
+ exit_pid=exit_pid)
del r
# Liberate the resource as soon as possible, to avoid holding onto
# open files or shared memory that is not needed anymore
del call_item
+ if exit_pid is not None:
+ return
+
class _ExecutorManagerThread(threading.Thread):
"""Manages the communication between this process and the worker processes.
@@ -305,6 +320,10 @@
# A queue.Queue of work ids e.g. Queue([5, 6, ...]).
self.work_ids_queue = executor._work_ids
+ # Maximum number of tasks a worker process can execute before
+ # exiting safely
+ self.max_tasks_per_child = executor._max_tasks_per_child
+
# A dict mapping work ids to _WorkItems e.g.
# {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
self.pending_work_items = executor._pending_work_items
@@ -324,15 +343,23 @@
return
if result_item is not None:
self.process_result_item(result_item)
+
+ process_exited = result_item.exit_pid is not None
+ if process_exited:
+ p = self.processes.pop(result_item.exit_pid)
+ p.join()
+
# Delete reference to result_item to avoid keeping references
# while waiting on new results.
del result_item
- # attempt to increment idle process count
- executor = self.executor_reference()
- if executor is not None:
- executor._idle_worker_semaphore.release()
- del executor
+ if executor := self.executor_reference():
+ if process_exited:
+ with self.shutdown_lock:
+ executor._adjust_process_count()
+ else:
+ executor._idle_worker_semaphore.release()
+ del executor
if self.is_shutting_down():
self.flag_executor_shutting_down()
@@ -392,7 +419,7 @@
result_item = result_reader.recv()
is_broken = False
except BaseException as e:
- cause = traceback.format_exception(type(e), e, e.__traceback__)
+ cause = format_exception(type(e), e, e.__traceback__)
elif wakeup_reader in ready:
is_broken = False
@@ -587,7 +614,7 @@
class ProcessPoolExecutor(_base.Executor):
def __init__(self, max_workers=None, mp_context=None,
- initializer=None, initargs=()):
+ initializer=None, initargs=(), *, max_tasks_per_child=None):
"""Initializes a new ProcessPoolExecutor instance.
Args:
@@ -595,9 +622,16 @@
execute the given calls. If None or not given then as many
worker processes will be created as the machine has processors.
mp_context: A multiprocessing context to launch the workers. This
- object should provide SimpleQueue, Queue and Process.
+ object should provide SimpleQueue, Queue and Process. Useful
+ to allow specific multiprocessing start methods.
initializer: A callable used to initialize worker processes.
initargs: A tuple of arguments to pass to the initializer.
+ max_tasks_per_child: The maximum number of tasks a worker process
+ can complete before it will exit and be replaced with a fresh
+ worker process. The default of None means worker process will
+ live as long as the executor. Requires a non-'fork' mp_context
+ start method. When given, we default to using 'spawn' if no
+ mp_context is supplied.
"""
_check_system_limits()
@@ -617,7 +651,10 @@
self._max_workers = max_workers
if mp_context is None:
- mp_context = mp.get_context()
+ if max_tasks_per_child is not None:
+ mp_context = mp.get_context("spawn")
+ else:
+ mp_context = mp.get_context()
self._mp_context = mp_context
# https://github.com/python/cpython/issues/90622
@@ -629,6 +666,18 @@
self._initializer = initializer
self._initargs = initargs
+ if max_tasks_per_child is not None:
+ if not isinstance(max_tasks_per_child, int):
+ raise TypeError("max_tasks_per_child must be an integer")
+ elif max_tasks_per_child <= 0:
+ raise ValueError("max_tasks_per_child must be >= 1")
+ if self._mp_context.get_start_method(allow_none=False) == "fork":
+ # https://github.com/python/cpython/issues/90622
+ raise ValueError("max_tasks_per_child is incompatible with"
+ " the 'fork' multiprocessing start method;"
+ " supply a different mp_context.")
+ self._max_tasks_per_child = max_tasks_per_child
+
# Management thread
self._executor_manager_thread = None
@@ -710,7 +759,8 @@
args=(self._call_queue,
self._result_queue,
self._initializer,
- self._initargs))
+ self._initargs,
+ self._max_tasks_per_child))
p.start()
self._processes[p.pid] = p
diff --git a/common/py3-stdlib/configparser.py b/common/py3-stdlib/configparser.py
index 7855270..df2d7e3 100644
--- a/common/py3-stdlib/configparser.py
+++ b/common/py3-stdlib/configparser.py
@@ -526,6 +526,15 @@
_KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ warnings.warn(
+ "LegacyInterpolation has been deprecated since Python 3.2 "
+ "and will be removed from the configparser module in Python 3.13. "
+ "Use BasicInterpolation or ExtendedInterpolation instead.",
+ DeprecationWarning, stacklevel=2
+ )
+
def before_get(self, parser, section, option, value, vars):
rawval = value
depth = MAX_INTERPOLATION_DEPTH
@@ -634,6 +643,11 @@
self._interpolation = self._DEFAULT_INTERPOLATION
if self._interpolation is None:
self._interpolation = Interpolation()
+ if not isinstance(self._interpolation, Interpolation):
+ raise TypeError(
+ f"interpolation= must be None or an instance of Interpolation;"
+ f" got an object of type {type(self._interpolation)}"
+ )
if converters is not _UNSET:
self._converters.update(converters)
if defaults:
diff --git a/common/py3-stdlib/contextlib.py b/common/py3-stdlib/contextlib.py
index c63a849..58e9a49 100644
--- a/common/py3-stdlib/contextlib.py
+++ b/common/py3-stdlib/contextlib.py
@@ -1,5 +1,6 @@
"""Utilities for with-statement contexts. See PEP 343."""
import abc
+import os
import sys
import _collections_abc
from collections import deque
@@ -9,7 +10,8 @@
__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
"AbstractContextManager", "AbstractAsyncContextManager",
"AsyncExitStack", "ContextDecorator", "ExitStack",
- "redirect_stdout", "redirect_stderr", "suppress", "aclosing"]
+ "redirect_stdout", "redirect_stderr", "suppress", "aclosing",
+ "chdir"]
class AbstractContextManager(abc.ABC):
@@ -159,6 +161,7 @@
except RuntimeError as exc:
# Don't re-raise the passed in exception. (issue27122)
if exc is value:
+ exc.__traceback__ = traceback
return False
# Avoid suppressing if a StopIteration exception
# was passed to throw() and later wrapped into a RuntimeError
@@ -170,6 +173,7 @@
isinstance(value, StopIteration)
and exc.__cause__ is value
):
+ value.__traceback__ = traceback
return False
raise
except BaseException as exc:
@@ -181,6 +185,7 @@
# and the __exit__() protocol.
if exc is not value:
raise
+ exc.__traceback__ = traceback
return False
raise RuntimeError("generator didn't stop after throw()")
@@ -223,17 +228,19 @@
except RuntimeError as exc:
# Don't re-raise the passed in exception. (issue27122)
if exc is value:
+ exc.__traceback__ = traceback
return False
# Avoid suppressing if a Stop(Async)Iteration exception
# was passed to athrow() and later wrapped into a RuntimeError
# (see PEP 479 for sync generators; async generators also
# have this behavior). But do this only if the exception wrapped
- # by the RuntimeError is actully Stop(Async)Iteration (see
+ # by the RuntimeError is actually Stop(Async)Iteration (see
# issue29692).
if (
isinstance(value, (StopIteration, StopAsyncIteration))
and exc.__cause__ is value
):
+ value.__traceback__ = traceback
return False
raise
except BaseException as exc:
@@ -245,6 +252,7 @@
# and the __exit__() protocol.
if exc is not value:
raise
+ exc.__traceback__ = traceback
return False
raise RuntimeError("generator didn't stop after athrow()")
@@ -487,9 +495,14 @@
"""
# We look up the special methods on the type to match the with
# statement.
- _cm_type = type(cm)
- _exit = _cm_type.__exit__
- result = _cm_type.__enter__(cm)
+ cls = type(cm)
+ try:
+ _enter = cls.__enter__
+ _exit = cls.__exit__
+ except AttributeError:
+ raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
+ f"not support the context manager protocol") from None
+ result = _enter(cm)
self._push_cm_exit(cm, _exit)
return result
@@ -614,9 +627,15 @@
If successful, also pushes its __aexit__ method as a callback and
returns the result of the __aenter__ method.
"""
- _cm_type = type(cm)
- _exit = _cm_type.__aexit__
- result = await _cm_type.__aenter__(cm)
+ cls = type(cm)
+ try:
+ _enter = cls.__aenter__
+ _exit = cls.__aexit__
+ except AttributeError:
+ raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
+ f"not support the asynchronous context manager protocol"
+ ) from None
+ result = await _enter(cm)
self._push_async_cm_exit(cm, _exit)
return result
@@ -743,3 +762,18 @@
async def __aexit__(self, *excinfo):
pass
+
+
+class chdir(AbstractContextManager):
+ """Non thread-safe context manager to change the current working directory."""
+
+ def __init__(self, path):
+ self.path = path
+ self._old_cwd = []
+
+ def __enter__(self):
+ self._old_cwd.append(os.getcwd())
+ os.chdir(self.path)
+
+ def __exit__(self, *excinfo):
+ os.chdir(self._old_cwd.pop())
diff --git a/common/py3-stdlib/copyreg.py b/common/py3-stdlib/copyreg.py
index 356db6f..c8a52a2 100644
--- a/common/py3-stdlib/copyreg.py
+++ b/common/py3-stdlib/copyreg.py
@@ -89,6 +89,10 @@
except AttributeError:
dict = None
else:
+ if (type(self).__getstate__ is object.__getstate__ and
+ getattr(self, "__slots__", None)):
+ raise TypeError("a class that defines __slots__ without "
+ "defining __getstate__ cannot be pickled")
dict = getstate()
if dict:
return _reconstructor, args, dict
diff --git a/common/py3-stdlib/crypt.py b/common/py3-stdlib/crypt.py
index b296c3e..de4a14a 100644
--- a/common/py3-stdlib/crypt.py
+++ b/common/py3-stdlib/crypt.py
@@ -12,10 +12,14 @@
import errno
import string as _string
+import warnings
from random import SystemRandom as _SystemRandom
from collections import namedtuple as _namedtuple
+warnings._deprecated(__name__, remove=(3, 13))
+
+
_saltchars = _string.ascii_letters + _string.digits + './'
_sr = _SystemRandom()
diff --git a/common/py3-stdlib/ctypes/__init__.py b/common/py3-stdlib/ctypes/__init__.py
index 4afa4eb..26135ad 100644
--- a/common/py3-stdlib/ctypes/__init__.py
+++ b/common/py3-stdlib/ctypes/__init__.py
@@ -65,12 +65,8 @@
return buf
raise TypeError(init)
-def c_buffer(init, size=None):
-## "deprecated, use create_string_buffer instead"
-## import warnings
-## warnings.warn("c_buffer is deprecated, use create_string_buffer instead",
-## DeprecationWarning, stacklevel=2)
- return create_string_buffer(init, size)
+# Alias to create_string_buffer() for backward compatibility
+c_buffer = create_string_buffer
_c_functype_cache = {}
def CFUNCTYPE(restype, *argtypes, **kw):
@@ -96,15 +92,18 @@
flags |= _FUNCFLAG_USE_LASTERROR
if kw:
raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
+
try:
return _c_functype_cache[(restype, argtypes, flags)]
except KeyError:
- class CFunctionType(_CFuncPtr):
- _argtypes_ = argtypes
- _restype_ = restype
- _flags_ = flags
- _c_functype_cache[(restype, argtypes, flags)] = CFunctionType
- return CFunctionType
+ pass
+
+ class CFunctionType(_CFuncPtr):
+ _argtypes_ = argtypes
+ _restype_ = restype
+ _flags_ = flags
+ _c_functype_cache[(restype, argtypes, flags)] = CFunctionType
+ return CFunctionType
if _os.name == "nt":
from _ctypes import LoadLibrary as _dlopen
@@ -120,15 +119,18 @@
flags |= _FUNCFLAG_USE_LASTERROR
if kw:
raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
+
try:
return _win_functype_cache[(restype, argtypes, flags)]
except KeyError:
- class WinFunctionType(_CFuncPtr):
- _argtypes_ = argtypes
- _restype_ = restype
- _flags_ = flags
- _win_functype_cache[(restype, argtypes, flags)] = WinFunctionType
- return WinFunctionType
+ pass
+
+ class WinFunctionType(_CFuncPtr):
+ _argtypes_ = argtypes
+ _restype_ = restype
+ _flags_ = flags
+ _win_functype_cache[(restype, argtypes, flags)] = WinFunctionType
+ return WinFunctionType
if WINFUNCTYPE.__doc__:
WINFUNCTYPE.__doc__ = CFUNCTYPE.__doc__.replace("CFUNCTYPE", "WINFUNCTYPE")
@@ -546,6 +548,7 @@
return ccom.DllCanUnloadNow()
from ctypes._endian import BigEndianStructure, LittleEndianStructure
+from ctypes._endian import BigEndianUnion, LittleEndianUnion
# Fill in specifically-sized types
c_int8 = c_byte
diff --git a/common/py3-stdlib/ctypes/_endian.py b/common/py3-stdlib/ctypes/_endian.py
index 37444bd..34dee64 100644
--- a/common/py3-stdlib/ctypes/_endian.py
+++ b/common/py3-stdlib/ctypes/_endian.py
@@ -20,7 +20,7 @@
return typ
raise TypeError("This type does not support other endian: %s" % typ)
-class _swapped_meta(type(Structure)):
+class _swapped_meta:
def __setattr__(self, attrname, value):
if attrname == "_fields_":
fields = []
@@ -31,6 +31,8 @@
fields.append((name, _other_endian(typ)) + rest)
value = fields
super().__setattr__(attrname, value)
+class _swapped_struct_meta(_swapped_meta, type(Structure)): pass
+class _swapped_union_meta(_swapped_meta, type(Union)): pass
################################################################
@@ -43,19 +45,34 @@
LittleEndianStructure = Structure
- class BigEndianStructure(Structure, metaclass=_swapped_meta):
+ class BigEndianStructure(Structure, metaclass=_swapped_struct_meta):
"""Structure with big endian byte order"""
__slots__ = ()
_swappedbytes_ = None
+ LittleEndianUnion = Union
+
+ class BigEndianUnion(Union, metaclass=_swapped_union_meta):
+ """Union with big endian byte order"""
+ __slots__ = ()
+ _swappedbytes_ = None
+
elif sys.byteorder == "big":
_OTHER_ENDIAN = "__ctype_le__"
BigEndianStructure = Structure
- class LittleEndianStructure(Structure, metaclass=_swapped_meta):
+
+ class LittleEndianStructure(Structure, metaclass=_swapped_struct_meta):
"""Structure with little endian byte order"""
__slots__ = ()
_swappedbytes_ = None
+ BigEndianUnion = Union
+
+ class LittleEndianUnion(Union, metaclass=_swapped_union_meta):
+ """Union with little endian byte order"""
+ __slots__ = ()
+ _swappedbytes_ = None
+
else:
raise RuntimeError("Invalid byteorder")
diff --git a/common/py3-stdlib/ctypes/macholib/dyld.py b/common/py3-stdlib/ctypes/macholib/dyld.py
index 1c3f8fd..583c47d 100644
--- a/common/py3-stdlib/ctypes/macholib/dyld.py
+++ b/common/py3-stdlib/ctypes/macholib/dyld.py
@@ -163,11 +163,3 @@
raise error
finally:
error = None
-
-def test_dyld_find():
- env = {}
- assert dyld_find('libSystem.dylib') == '/usr/lib/libSystem.dylib'
- assert dyld_find('System.framework/System') == '/System/Library/Frameworks/System.framework/System'
-
-if __name__ == '__main__':
- test_dyld_find()
diff --git a/common/py3-stdlib/ctypes/macholib/dylib.py b/common/py3-stdlib/ctypes/macholib/dylib.py
index aa10750..0ad4cba 100644
--- a/common/py3-stdlib/ctypes/macholib/dylib.py
+++ b/common/py3-stdlib/ctypes/macholib/dylib.py
@@ -40,24 +40,3 @@
if not is_dylib:
return None
return is_dylib.groupdict()
-
-
-def test_dylib_info():
- def d(location=None, name=None, shortname=None, version=None, suffix=None):
- return dict(
- location=location,
- name=name,
- shortname=shortname,
- version=version,
- suffix=suffix
- )
- assert dylib_info('completely/invalid') is None
- assert dylib_info('completely/invalide_debug') is None
- assert dylib_info('P/Foo.dylib') == d('P', 'Foo.dylib', 'Foo')
- assert dylib_info('P/Foo_debug.dylib') == d('P', 'Foo_debug.dylib', 'Foo', suffix='debug')
- assert dylib_info('P/Foo.A.dylib') == d('P', 'Foo.A.dylib', 'Foo', 'A')
- assert dylib_info('P/Foo_debug.A.dylib') == d('P', 'Foo_debug.A.dylib', 'Foo_debug', 'A')
- assert dylib_info('P/Foo.A_debug.dylib') == d('P', 'Foo.A_debug.dylib', 'Foo', 'A', 'debug')
-
-if __name__ == '__main__':
- test_dylib_info()
diff --git a/common/py3-stdlib/ctypes/macholib/framework.py b/common/py3-stdlib/ctypes/macholib/framework.py
index ad6ed55..495679f 100644
--- a/common/py3-stdlib/ctypes/macholib/framework.py
+++ b/common/py3-stdlib/ctypes/macholib/framework.py
@@ -40,26 +40,3 @@
if not is_framework:
return None
return is_framework.groupdict()
-
-def test_framework_info():
- def d(location=None, name=None, shortname=None, version=None, suffix=None):
- return dict(
- location=location,
- name=name,
- shortname=shortname,
- version=version,
- suffix=suffix
- )
- assert framework_info('completely/invalid') is None
- assert framework_info('completely/invalid/_debug') is None
- assert framework_info('P/F.framework') is None
- assert framework_info('P/F.framework/_debug') is None
- assert framework_info('P/F.framework/F') == d('P', 'F.framework/F', 'F')
- assert framework_info('P/F.framework/F_debug') == d('P', 'F.framework/F_debug', 'F', suffix='debug')
- assert framework_info('P/F.framework/Versions') is None
- assert framework_info('P/F.framework/Versions/A') is None
- assert framework_info('P/F.framework/Versions/A/F') == d('P', 'F.framework/Versions/A/F', 'F', 'A')
- assert framework_info('P/F.framework/Versions/A/F_debug') == d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug')
-
-if __name__ == '__main__':
- test_framework_info()
diff --git a/common/py3-stdlib/ctypes/test/test_as_parameter.py b/common/py3-stdlib/ctypes/test/test_as_parameter.py
index f9d27cb..9c39179 100644
--- a/common/py3-stdlib/ctypes/test/test_as_parameter.py
+++ b/common/py3-stdlib/ctypes/test/test_as_parameter.py
@@ -122,6 +122,7 @@
result = f(self.wrap(-10), self.wrap(cb))
self.assertEqual(result, -18)
+ @need_symbol('c_longlong')
def test_longlong_callbacks(self):
f = dll._testfunc_callback_q_qf
diff --git a/common/py3-stdlib/ctypes/test/test_byteswap.py b/common/py3-stdlib/ctypes/test/test_byteswap.py
index 01c97e8..7e98559 100644
--- a/common/py3-stdlib/ctypes/test/test_byteswap.py
+++ b/common/py3-stdlib/ctypes/test/test_byteswap.py
@@ -170,40 +170,34 @@
self.assertIs(c_char.__ctype_le__, c_char)
self.assertIs(c_char.__ctype_be__, c_char)
- def test_struct_fields_1(self):
- if sys.byteorder == "little":
- base = BigEndianStructure
- else:
- base = LittleEndianStructure
+ def test_struct_fields_unsupported_byte_order(self):
- class T(base):
- pass
- _fields_ = [("a", c_ubyte),
- ("b", c_byte),
- ("c", c_short),
- ("d", c_ushort),
- ("e", c_int),
- ("f", c_uint),
- ("g", c_long),
- ("h", c_ulong),
- ("i", c_longlong),
- ("k", c_ulonglong),
- ("l", c_float),
- ("m", c_double),
- ("n", c_char),
-
- ("b1", c_byte, 3),
- ("b2", c_byte, 3),
- ("b3", c_byte, 2),
- ("a", c_int * 3 * 3 * 3)]
- T._fields_ = _fields_
+ fields = [
+ ("a", c_ubyte),
+ ("b", c_byte),
+ ("c", c_short),
+ ("d", c_ushort),
+ ("e", c_int),
+ ("f", c_uint),
+ ("g", c_long),
+ ("h", c_ulong),
+ ("i", c_longlong),
+ ("k", c_ulonglong),
+ ("l", c_float),
+ ("m", c_double),
+ ("n", c_char),
+ ("b1", c_byte, 3),
+ ("b2", c_byte, 3),
+ ("b3", c_byte, 2),
+ ("a", c_int * 3 * 3 * 3)
+ ]
# these fields do not support different byte order:
for typ in c_wchar, c_void_p, POINTER(c_int):
- _fields_.append(("x", typ))
- class T(base):
- pass
- self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
+ with self.assertRaises(TypeError):
+ class T(BigEndianStructure if sys.byteorder == "little" else LittleEndianStructure):
+ _fields_ = fields + [("x", typ)]
+
def test_struct_struct(self):
# nested structures with different byteorders
@@ -233,7 +227,7 @@
self.assertEqual(s.point.x, 1)
self.assertEqual(s.point.y, 2)
- def test_struct_fields_2(self):
+ def test_struct_field_alignment(self):
# standard packing in struct uses no alignment.
# So, we have to align using pad bytes.
#
@@ -267,7 +261,6 @@
class S(base):
_pack_ = 1
_fields_ = [("b", c_byte),
-
("h", c_short),
("_1", c_byte),
@@ -311,5 +304,61 @@
s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
self.assertEqual(bin(s1), bin(s2))
+ def test_union_fields_unsupported_byte_order(self):
+
+ fields = [
+ ("a", c_ubyte),
+ ("b", c_byte),
+ ("c", c_short),
+ ("d", c_ushort),
+ ("e", c_int),
+ ("f", c_uint),
+ ("g", c_long),
+ ("h", c_ulong),
+ ("i", c_longlong),
+ ("k", c_ulonglong),
+ ("l", c_float),
+ ("m", c_double),
+ ("n", c_char),
+ ("b1", c_byte, 3),
+ ("b2", c_byte, 3),
+ ("b3", c_byte, 2),
+ ("a", c_int * 3 * 3 * 3)
+ ]
+
+ # these fields do not support different byte order:
+ for typ in c_wchar, c_void_p, POINTER(c_int):
+ with self.assertRaises(TypeError):
+ class T(BigEndianUnion if sys.byteorder == "little" else LittleEndianUnion):
+ _fields_ = fields + [("x", typ)]
+
+ def test_union_struct(self):
+ # nested structures in unions with different byteorders
+
+ # create nested structures in unions with given byteorders and set memory to data
+
+ for nested, data in (
+ (BigEndianStructure, b'\0\0\0\1\0\0\0\2'),
+ (LittleEndianStructure, b'\1\0\0\0\2\0\0\0'),
+ ):
+ for parent in (
+ BigEndianUnion,
+ LittleEndianUnion,
+ Union,
+ ):
+ class NestedStructure(nested):
+ _fields_ = [("x", c_uint32),
+ ("y", c_uint32)]
+
+ class TestUnion(parent):
+ _fields_ = [("point", NestedStructure)]
+
+ self.assertEqual(len(data), sizeof(TestUnion))
+ ptr = POINTER(TestUnion)
+ s = cast(data, ptr)[0]
+ del ctypes._pointer_type_cache[TestUnion]
+ self.assertEqual(s.point.x, 1)
+ self.assertEqual(s.point.y, 2)
+
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/ctypes/test/test_callbacks.py b/common/py3-stdlib/ctypes/test/test_callbacks.py
index d8e9c5a..8f95a24 100644
--- a/common/py3-stdlib/ctypes/test/test_callbacks.py
+++ b/common/py3-stdlib/ctypes/test/test_callbacks.py
@@ -4,6 +4,7 @@
from ctypes import *
from ctypes.test import need_symbol
+from _ctypes import CTYPES_MAX_ARGCOUNT
import _ctypes_test
class Callbacks(unittest.TestCase):
@@ -64,10 +65,12 @@
def test_ulong(self):
self.check_type(c_ulong, 42)
+ @need_symbol('c_longlong')
def test_longlong(self):
self.check_type(c_longlong, 42)
self.check_type(c_longlong, -42)
+ @need_symbol('c_ulonglong')
def test_ulonglong(self):
self.check_type(c_ulonglong, 42)
@@ -81,6 +84,7 @@
self.check_type(c_double, 3.14)
self.check_type(c_double, -3.14)
+ @need_symbol('c_longdouble')
def test_longdouble(self):
self.check_type(c_longdouble, 3.14)
self.check_type(c_longdouble, -3.14)
@@ -293,16 +297,21 @@
def func(*args):
return len(args)
- CTYPES_MAX_ARGCOUNT = 1024
+ # valid call with nargs <= CTYPES_MAX_ARGCOUNT
proto = CFUNCTYPE(c_int, *(c_int,) * CTYPES_MAX_ARGCOUNT)
cb = proto(func)
args1 = (1,) * CTYPES_MAX_ARGCOUNT
self.assertEqual(cb(*args1), CTYPES_MAX_ARGCOUNT)
+ # invalid call with nargs > CTYPES_MAX_ARGCOUNT
args2 = (1,) * (CTYPES_MAX_ARGCOUNT + 1)
with self.assertRaises(ArgumentError):
cb(*args2)
+ # error when creating the type with too many arguments
+ with self.assertRaises(ArgumentError):
+ CFUNCTYPE(c_int, *(c_int,) * (CTYPES_MAX_ARGCOUNT + 1))
+
def test_convert_result_error(self):
def func():
return ("tuple",)
diff --git a/common/py3-stdlib/ctypes/test/test_cfuncs.py b/common/py3-stdlib/ctypes/test/test_cfuncs.py
index ac2240f..09b0684 100644
--- a/common/py3-stdlib/ctypes/test/test_cfuncs.py
+++ b/common/py3-stdlib/ctypes/test/test_cfuncs.py
@@ -111,24 +111,28 @@
self.assertEqual(self._dll.tf_bL(b' ', 4294967295), 1431655765)
self.assertEqual(self.U(), 4294967295)
+ @need_symbol('c_longlong')
def test_longlong(self):
self._dll.tf_q.restype = c_longlong
self._dll.tf_q.argtypes = (c_longlong, )
self.assertEqual(self._dll.tf_q(-9223372036854775806), -3074457345618258602)
self.assertEqual(self.S(), -9223372036854775806)
+ @need_symbol('c_longlong')
def test_longlong_plus(self):
self._dll.tf_bq.restype = c_longlong
self._dll.tf_bq.argtypes = (c_byte, c_longlong)
self.assertEqual(self._dll.tf_bq(0, -9223372036854775806), -3074457345618258602)
self.assertEqual(self.S(), -9223372036854775806)
+ @need_symbol('c_ulonglong')
def test_ulonglong(self):
self._dll.tf_Q.restype = c_ulonglong
self._dll.tf_Q.argtypes = (c_ulonglong, )
self.assertEqual(self._dll.tf_Q(18446744073709551615), 6148914691236517205)
self.assertEqual(self.U(), 18446744073709551615)
+ @need_symbol('c_ulonglong')
def test_ulonglong_plus(self):
self._dll.tf_bQ.restype = c_ulonglong
self._dll.tf_bQ.argtypes = (c_byte, c_ulonglong)
@@ -159,12 +163,14 @@
self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
self.assertEqual(self.S(), 42)
+ @need_symbol('c_longdouble')
def test_longdouble(self):
self._dll.tf_D.restype = c_longdouble
self._dll.tf_D.argtypes = (c_longdouble,)
self.assertEqual(self._dll.tf_D(42.), 14.)
self.assertEqual(self.S(), 42)
+ @need_symbol('c_longdouble')
def test_longdouble_plus(self):
self._dll.tf_bD.restype = c_longdouble
self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/common/py3-stdlib/ctypes/test/test_functions.py b/common/py3-stdlib/ctypes/test/test_functions.py
index bdb044e..fc57170 100644
--- a/common/py3-stdlib/ctypes/test/test_functions.py
+++ b/common/py3-stdlib/ctypes/test/test_functions.py
@@ -128,6 +128,7 @@
self.assertEqual(result, -21)
self.assertEqual(type(result), float)
+ @need_symbol('c_longdouble')
def test_longdoubleresult(self):
f = dll._testfunc_D_bhilfD
f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
@@ -199,15 +200,6 @@
result = f(byref(c_int(99)))
self.assertNotEqual(result.contents, 99)
- def test_errors(self):
- f = dll._testfunc_p_p
- f.restype = c_int
-
- class X(Structure):
- _fields_ = [("y", c_int)]
-
- self.assertRaises(TypeError, f, X()) #cannot convert parameter
-
################################################################
def test_shorts(self):
f = dll._testfunc_callback_i_if
diff --git a/common/py3-stdlib/ctypes/test/test_macholib.py b/common/py3-stdlib/ctypes/test/test_macholib.py
index a1bac26..bc75f1a 100644
--- a/common/py3-stdlib/ctypes/test/test_macholib.py
+++ b/common/py3-stdlib/ctypes/test/test_macholib.py
@@ -32,6 +32,8 @@
# -bob
from ctypes.macholib.dyld import dyld_find
+from ctypes.macholib.dylib import dylib_info
+from ctypes.macholib.framework import framework_info
def find_lib(name):
possible = ['lib'+name+'.dylib', name+'.dylib', name+'.framework/'+name]
@@ -42,9 +44,20 @@
pass
raise ValueError("%s not found" % (name,))
+
+def d(location=None, name=None, shortname=None, version=None, suffix=None):
+ return {'location': location, 'name': name, 'shortname': shortname,
+ 'version': version, 'suffix': suffix}
+
+
class MachOTest(unittest.TestCase):
@unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
def test_find(self):
+ self.assertEqual(dyld_find('libSystem.dylib'),
+ '/usr/lib/libSystem.dylib')
+ self.assertEqual(dyld_find('System.framework/System'),
+ '/System/Library/Frameworks/System.framework/System')
+
# On Mac OS 11, system dylibs are only present in the shared cache,
# so symlinks like libpthread.dylib -> libSystem.B.dylib will not
# be resolved by dyld_find
@@ -62,5 +75,36 @@
('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
'/System/Library/Frameworks/IOKit.framework/IOKit'))
+ @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
+ def test_info(self):
+ self.assertIsNone(dylib_info('completely/invalid'))
+ self.assertIsNone(dylib_info('completely/invalide_debug'))
+ self.assertEqual(dylib_info('P/Foo.dylib'), d('P', 'Foo.dylib', 'Foo'))
+ self.assertEqual(dylib_info('P/Foo_debug.dylib'),
+ d('P', 'Foo_debug.dylib', 'Foo', suffix='debug'))
+ self.assertEqual(dylib_info('P/Foo.A.dylib'),
+ d('P', 'Foo.A.dylib', 'Foo', 'A'))
+ self.assertEqual(dylib_info('P/Foo_debug.A.dylib'),
+ d('P', 'Foo_debug.A.dylib', 'Foo_debug', 'A'))
+ self.assertEqual(dylib_info('P/Foo.A_debug.dylib'),
+ d('P', 'Foo.A_debug.dylib', 'Foo', 'A', 'debug'))
+
+ @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
+ def test_framework_info(self):
+ self.assertIsNone(framework_info('completely/invalid'))
+ self.assertIsNone(framework_info('completely/invalid/_debug'))
+ self.assertIsNone(framework_info('P/F.framework'))
+ self.assertIsNone(framework_info('P/F.framework/_debug'))
+ self.assertEqual(framework_info('P/F.framework/F'),
+ d('P', 'F.framework/F', 'F'))
+ self.assertEqual(framework_info('P/F.framework/F_debug'),
+ d('P', 'F.framework/F_debug', 'F', suffix='debug'))
+ self.assertIsNone(framework_info('P/F.framework/Versions'))
+ self.assertIsNone(framework_info('P/F.framework/Versions/A'))
+ self.assertEqual(framework_info('P/F.framework/Versions/A/F'),
+ d('P', 'F.framework/Versions/A/F', 'F', 'A'))
+ self.assertEqual(framework_info('P/F.framework/Versions/A/F_debug'),
+ d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug'))
+
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/ctypes/test/test_strings.py b/common/py3-stdlib/ctypes/test/test_strings.py
index 5434efd..12e2088 100644
--- a/common/py3-stdlib/ctypes/test/test_strings.py
+++ b/common/py3-stdlib/ctypes/test/test_strings.py
@@ -85,74 +85,6 @@
w = c_wchar(u)
self.assertEqual(w.value, u)
-class StringTestCase(unittest.TestCase):
- @unittest.skip('test disabled')
- def test_basic_strings(self):
- cs = c_string("abcdef")
-
- # Cannot call len on a c_string any longer
- self.assertRaises(TypeError, len, cs)
- self.assertEqual(sizeof(cs), 7)
-
- # The value property is the string up to the first terminating NUL.
- self.assertEqual(cs.value, "abcdef")
- self.assertEqual(c_string("abc\000def").value, "abc")
-
- # The raw property is the total buffer contents:
- self.assertEqual(cs.raw, "abcdef\000")
- self.assertEqual(c_string("abc\000def").raw, "abc\000def\000")
-
- # We can change the value:
- cs.value = "ab"
- self.assertEqual(cs.value, "ab")
- self.assertEqual(cs.raw, "ab\000\000\000\000\000")
-
- cs.raw = "XY"
- self.assertEqual(cs.value, "XY")
- self.assertEqual(cs.raw, "XY\000\000\000\000\000")
-
- self.assertRaises(TypeError, c_string, "123")
-
- @unittest.skip('test disabled')
- def test_sized_strings(self):
-
- # New in releases later than 0.4.0:
- self.assertRaises(TypeError, c_string, None)
-
- # New in releases later than 0.4.0:
- # c_string(number) returns an empty string of size number
- self.assertEqual(len(c_string(32).raw), 32)
- self.assertRaises(ValueError, c_string, -1)
- self.assertRaises(ValueError, c_string, 0)
-
- # These tests fail, because it is no longer initialized
-## self.assertEqual(c_string(2).value, "")
-## self.assertEqual(c_string(2).raw, "\000\000")
- self.assertEqual(c_string(2).raw[-1], "\000")
- self.assertEqual(len(c_string(2).raw), 2)
-
- @unittest.skip('test disabled')
- def test_initialized_strings(self):
-
- self.assertEqual(c_string("ab", 4).raw[:2], "ab")
- self.assertEqual(c_string("ab", 4).raw[:2:], "ab")
- self.assertEqual(c_string("ab", 4).raw[:2:-1], "ba")
- self.assertEqual(c_string("ab", 4).raw[:2:2], "a")
- self.assertEqual(c_string("ab", 4).raw[-1], "\000")
- self.assertEqual(c_string("ab", 2).raw, "a\000")
-
- @unittest.skip('test disabled')
- def test_toolong(self):
- cs = c_string("abcdef")
- # Much too long string:
- self.assertRaises(ValueError, setattr, cs, "value", "123456789012345")
-
- # One char too long values:
- self.assertRaises(ValueError, setattr, cs, "value", "1234567")
-
- @unittest.skip('test disabled')
- def test_perf(self):
- check_perf()
@need_symbol('c_wchar')
class WStringTestCase(unittest.TestCase):
@@ -208,25 +140,6 @@
stop = clock()
print("%20s: %.2f us" % (msg, ((stop-start)*1e6/5/rep)))
-def check_perf():
- # Construct 5 objects
-
- REP = 200000
-
- run_test(REP, "c_string(None)", c_string, None)
- run_test(REP, "c_string('abc')", c_string, 'abc')
-
-# Python 2.3 -OO, win2k, P4 700 MHz:
-#
-# c_string(None): 1.75 us
-# c_string('abc'): 2.74 us
-
-# Python 2.2 -OO, win2k, P4 700 MHz:
-#
-# c_string(None): 2.95 us
-# c_string('abc'): 3.67 us
-
if __name__ == '__main__':
-## check_perf()
unittest.main()
diff --git a/common/py3-stdlib/ctypes/test/test_struct_fields.py b/common/py3-stdlib/ctypes/test/test_struct_fields.py
index fefeaea..e444f5e 100644
--- a/common/py3-stdlib/ctypes/test/test_struct_fields.py
+++ b/common/py3-stdlib/ctypes/test/test_struct_fields.py
@@ -54,6 +54,12 @@
x.char = b'a\0b\0'
self.assertEqual(bytes(x), b'a\x00###')
+ def test_6(self):
+ class X(Structure):
+ _fields_ = [("x", c_int)]
+ CField = type(X.x)
+ self.assertRaises(TypeError, CField)
+
def test_gh99275(self):
class BrokenStructure(Structure):
def __init_subclass__(cls, **kwargs):
diff --git a/common/py3-stdlib/ctypes/test/test_values.py b/common/py3-stdlib/ctypes/test/test_values.py
index 7514fe8..435fdd2 100644
--- a/common/py3-stdlib/ctypes/test/test_values.py
+++ b/common/py3-stdlib/ctypes/test/test_values.py
@@ -2,9 +2,12 @@
A testcase which accesses *values* in a dll.
"""
+import _imp
+import importlib.util
import unittest
import sys
from ctypes import *
+from test.support import import_helper
import _ctypes_test
@@ -50,45 +53,44 @@
class struct_frozen(Structure):
_fields_ = [("name", c_char_p),
("code", POINTER(c_ubyte)),
- ("size", c_int)]
+ ("size", c_int),
+ ("is_package", c_int),
+ ("get_code", POINTER(c_ubyte)), # Function ptr
+ ]
FrozenTable = POINTER(struct_frozen)
- ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules")
- # ft is a pointer to the struct_frozen entries:
- items = []
- # _frozen_importlib changes size whenever importlib._bootstrap
- # changes, so it gets a special case. We should make sure it's
- # found, but don't worry about its size too much. The same
- # applies to _frozen_importlib_external.
- bootstrap_seen = []
- bootstrap_expected = [
- b'_frozen_importlib',
- b'_frozen_importlib_external',
- b'zipimport',
- ]
- for entry in ft:
- # This is dangerous. We *can* iterate over a pointer, but
- # the loop will not terminate (maybe with an access
- # violation;-) because the pointer instance has no size.
- if entry.name is None:
- break
+ modules = []
+ for group in ["Bootstrap", "Stdlib", "Test"]:
+ ft = FrozenTable.in_dll(pythonapi, f"_PyImport_Frozen{group}")
+ # ft is a pointer to the struct_frozen entries:
+ for entry in ft:
+ # This is dangerous. We *can* iterate over a pointer, but
+ # the loop will not terminate (maybe with an access
+ # violation;-) because the pointer instance has no size.
+ if entry.name is None:
+ break
+ modname = entry.name.decode("ascii")
+ modules.append(modname)
+ with self.subTest(modname):
+ if entry.size != 0:
+ # Do a sanity check on entry.size and entry.code.
+ self.assertGreater(abs(entry.size), 10)
+ self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
+ # Check the module's package-ness.
+ with import_helper.frozen_modules():
+ spec = importlib.util.find_spec(modname)
+ if entry.is_package:
+ # It's a package.
+ self.assertIsNotNone(spec.submodule_search_locations)
+ else:
+ self.assertIsNone(spec.submodule_search_locations)
- if entry.name in bootstrap_expected:
- bootstrap_seen.append(entry.name)
- self.assertTrue(entry.size,
- "{!r} was reported as having no size".format(entry.name))
- continue
- items.append((entry.name.decode("ascii"), entry.size))
-
- expected = [("__hello__", 137),
- ("__phello__", -137),
- ("__phello__.spam", 137),
- ]
- self.assertEqual(items, expected, "PyImport_FrozenModules example "
- "in Doc/library/ctypes.rst may be out of date")
-
- self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
- "frozen bootstrap modules did not match PyImport_FrozenModules")
+ with import_helper.frozen_modules():
+ expected = _imp._frozen_module_names()
+ self.maxDiff = None
+ self.assertEqual(modules, expected,
+ "_PyImport_FrozenBootstrap example "
+ "in Doc/library/ctypes.rst may be out of date")
from ctypes import _pointer_type_cache
del _pointer_type_cache[struct_frozen]
diff --git a/common/py3-stdlib/dataclasses.py b/common/py3-stdlib/dataclasses.py
index 1742900..148daf4 100644
--- a/common/py3-stdlib/dataclasses.py
+++ b/common/py3-stdlib/dataclasses.py
@@ -6,6 +6,7 @@
import keyword
import builtins
import functools
+import itertools
import abc
import _thread
from types import FunctionType, GenericAlias
@@ -249,7 +250,7 @@
self.type = type
def __repr__(self):
- if isinstance(self.type, type) and not isinstance(self.type, GenericAlias):
+ if isinstance(self.type, type):
type_name = self.type.__name__
else:
# typing objects, e.g. List[int]
@@ -318,7 +319,7 @@
# This is used to support the PEP 487 __set_name__ protocol in the
# case where we're using a field that contains a descriptor as a
# default value. For details on __set_name__, see
- # https://www.python.org/dev/peps/pep-0487/#implementation-details.
+ # https://peps.python.org/pep-0487/#implementation-details.
#
# Note that in _process_class, this Field object is overwritten
# with the default value, so the end result is a descriptor that
@@ -807,8 +808,10 @@
raise TypeError(f'field {f.name} is a ClassVar but specifies '
'kw_only')
- # For real fields, disallow mutable defaults for known types.
- if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)):
+ # For real fields, disallow mutable defaults. Use unhashable as a proxy
+ # indicator for mutability. Read the __hash__ attribute from the class,
+ # not the instance.
+ if f._field_type is _FIELD and f.default.__class__.__hash__ is None:
raise ValueError(f'mutable default {type(f.default)} for field '
f'{f.name} is not allowed: use default_factory')
@@ -879,7 +882,7 @@
def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen,
- match_args, kw_only, slots):
+ match_args, kw_only, slots, weakref_slot):
# Now that dicts retain insertion order, there's no reason to use
# an ordered dict. I am leveraging that ordering here, because
# derived class fields overwrite base class fields, but the order
@@ -1089,16 +1092,24 @@
if not getattr(cls, '__doc__'):
# Create a class doc-string.
- cls.__doc__ = (cls.__name__ +
- str(inspect.signature(cls)).replace(' -> None', ''))
+ try:
+ # In some cases fetching a signature is not possible.
+ # But, we surely should not fail in this case.
+ text_sig = str(inspect.signature(cls)).replace(' -> None', '')
+ except (TypeError, ValueError):
+ text_sig = ''
+ cls.__doc__ = (cls.__name__ + text_sig)
if match_args:
# I could probably compute this once
_set_new_attribute(cls, '__match_args__',
tuple(f.name for f in std_init_fields))
+ # It's an error to specify weakref_slot if slots is False.
+ if weakref_slot and not slots:
+ raise TypeError('weakref_slot is True but slots is False')
if slots:
- cls = _add_slots(cls, frozen)
+ cls = _add_slots(cls, frozen, weakref_slot)
abc.update_abstractmethods(cls)
@@ -1106,7 +1117,7 @@
# _dataclass_getstate and _dataclass_setstate are needed for pickling frozen
-# classes with slots. These could be slighly more performant if we generated
+# classes with slots. These could be slightly more performant if we generated
# the code instead of iterating over fields. But that can be a project for
# another day, if performance becomes an issue.
def _dataclass_getstate(self):
@@ -1119,7 +1130,21 @@
object.__setattr__(self, field.name, value)
-def _add_slots(cls, is_frozen):
+def _get_slots(cls):
+ match cls.__dict__.get('__slots__'):
+ case None:
+ return
+ case str(slot):
+ yield slot
+ # Slots may be any iterable, but we cannot handle an iterator
+ # because it will already be (partially) consumed.
+ case iterable if not hasattr(iterable, '__next__'):
+ yield from iterable
+ case _:
+ raise TypeError(f"Slots of '{cls.__name__}' cannot be determined")
+
+
+def _add_slots(cls, is_frozen, weakref_slot):
# Need to create a new class, since we can't set __slots__
# after a class has been created.
@@ -1130,7 +1155,23 @@
# Create a new dict for our new class.
cls_dict = dict(cls.__dict__)
field_names = tuple(f.name for f in fields(cls))
- cls_dict['__slots__'] = field_names
+ # Make sure slots don't overlap with those in base classes.
+ inherited_slots = set(
+ itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1]))
+ )
+ # The slots for our class. Remove slots from our base classes. Add
+ # '__weakref__' if weakref_slot was given, unless it is already present.
+ cls_dict["__slots__"] = tuple(
+ itertools.filterfalse(
+ inherited_slots.__contains__,
+ itertools.chain(
+ # gh-93521: '__weakref__' also needs to be filtered out if
+ # already present in inherited_slots
+ field_names, ('__weakref__',) if weakref_slot else ()
+ )
+ ),
+ )
+
for field_name in field_names:
# Remove our attributes, if present. They'll still be
# available in _MARKER.
@@ -1139,6 +1180,9 @@
# Remove __dict__ itself.
cls_dict.pop('__dict__', None)
+ # Clear existing `__weakref__` descriptor, it belongs to a previous type:
+ cls_dict.pop('__weakref__', None) # gh-102069
+
# And finally create the class.
qualname = getattr(cls, '__qualname__', None)
cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
@@ -1147,33 +1191,35 @@
if is_frozen:
# Need this for pickling frozen classes with slots.
- cls.__getstate__ = _dataclass_getstate
- cls.__setstate__ = _dataclass_setstate
+ if '__getstate__' not in cls_dict:
+ cls.__getstate__ = _dataclass_getstate
+ if '__setstate__' not in cls_dict:
+ cls.__setstate__ = _dataclass_setstate
return cls
def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False,
unsafe_hash=False, frozen=False, match_args=True,
- kw_only=False, slots=False):
- """Returns the same class as was passed in, with dunder methods
- added based on the fields defined in the class.
+ kw_only=False, slots=False, weakref_slot=False):
+ """Add dunder methods based on the fields defined in the class.
Examines PEP 526 __annotations__ to determine fields.
- If init is true, an __init__() method is added to the class. If
- repr is true, a __repr__() method is added. If order is true, rich
+ If init is true, an __init__() method is added to the class. If repr
+ is true, a __repr__() method is added. If order is true, rich
comparison dunder methods are added. If unsafe_hash is true, a
- __hash__() method function is added. If frozen is true, fields may
- not be assigned to after instance creation. If match_args is true,
- the __match_args__ tuple is added. If kw_only is true, then by
- default all fields are keyword-only. If slots is true, an
- __slots__ attribute is added.
+ __hash__() method is added. If frozen is true, fields may not be
+ assigned to after instance creation. If match_args is true, the
+ __match_args__ tuple is added. If kw_only is true, then by default
+ all fields are keyword-only. If slots is true, a new class with a
+ __slots__ attribute is returned.
"""
def wrap(cls):
return _process_class(cls, init, repr, eq, order, unsafe_hash,
- frozen, match_args, kw_only, slots)
+ frozen, match_args, kw_only, slots,
+ weakref_slot)
# See if we're being called as @dataclass or @dataclass().
if cls is None:
@@ -1210,7 +1256,7 @@
def is_dataclass(obj):
"""Returns True if obj is a dataclass or an instance of a
dataclass."""
- cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj)
+ cls = obj if isinstance(obj, type) else type(obj)
return hasattr(cls, _FIELDS)
@@ -1218,7 +1264,7 @@
"""Return the fields of a dataclass instance as a new dictionary mapping
field names to field values.
- Example usage:
+ Example usage::
@dataclass
class C:
@@ -1289,8 +1335,8 @@
x: int
y: int
- c = C(1, 2)
- assert astuple(c) == (1, 2)
+ c = C(1, 2)
+ assert astuple(c) == (1, 2)
If given, 'tuple_factory' will be used instead of built-in tuple.
The function applies recursively to field values that are
@@ -1332,17 +1378,18 @@
def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True,
repr=True, eq=True, order=False, unsafe_hash=False,
- frozen=False, match_args=True, kw_only=False, slots=False):
+ frozen=False, match_args=True, kw_only=False, slots=False,
+ weakref_slot=False):
"""Return a new dynamically created dataclass.
The dataclass name will be 'cls_name'. 'fields' is an iterable
of either (name), (name, type) or (name, type, Field) objects. If type is
omitted, use the string 'typing.Any'. Field objects are created by
- the equivalent of calling 'field(name, type [, Field-info])'.
+ the equivalent of calling 'field(name, type [, Field-info])'.::
C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,))
- is equivalent to:
+ is equivalent to::
@dataclass
class C(Base):
@@ -1399,13 +1446,14 @@
# Apply the normal decorator.
return dataclass(cls, init=init, repr=repr, eq=eq, order=order,
unsafe_hash=unsafe_hash, frozen=frozen,
- match_args=match_args, kw_only=kw_only, slots=slots)
+ match_args=match_args, kw_only=kw_only, slots=slots,
+ weakref_slot=weakref_slot)
def replace(obj, /, **changes):
"""Return a new object replacing specified fields with new values.
- This is especially useful for frozen classes. Example usage:
+ This is especially useful for frozen classes. Example usage::
@dataclass(frozen=True)
class C:
@@ -1415,7 +1463,7 @@
c = C(1, 2)
c1 = replace(c, x=3)
assert c1.x == 3 and c1.y == 2
- """
+ """
# We're going to mutate 'changes', but that's okay because it's a
# new dict, even if called with 'replace(obj, **my_changes)'.
diff --git a/common/py3-stdlib/datetime.py b/common/py3-stdlib/datetime.py
index d087c98..c3c2568 100644
--- a/common/py3-stdlib/datetime.py
+++ b/common/py3-stdlib/datetime.py
@@ -5,7 +5,7 @@
"""
__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo",
- "MINYEAR", "MAXYEAR")
+ "MINYEAR", "MAXYEAR", "UTC")
import time as _time
@@ -262,58 +262,150 @@
return _time.strftime(newformat, timetuple)
# Helpers for parsing the result of isoformat()
+def _is_ascii_digit(c):
+ return c in "0123456789"
+
+def _find_isoformat_datetime_separator(dtstr):
+ # See the comment in _datetimemodule.c:_find_isoformat_datetime_separator
+ len_dtstr = len(dtstr)
+ if len_dtstr == 7:
+ return 7
+
+ assert len_dtstr > 7
+ date_separator = "-"
+ week_indicator = "W"
+
+ if dtstr[4] == date_separator:
+ if dtstr[5] == week_indicator:
+ if len_dtstr < 8:
+ raise ValueError("Invalid ISO string")
+ if len_dtstr > 8 and dtstr[8] == date_separator:
+ if len_dtstr == 9:
+ raise ValueError("Invalid ISO string")
+ if len_dtstr > 10 and _is_ascii_digit(dtstr[10]):
+ # This is as far as we need to resolve the ambiguity for
+ # the moment - if we have YYYY-Www-##, the separator is
+ # either a hyphen at 8 or a number at 10.
+ #
+ # We'll assume it's a hyphen at 8 because it's way more
+ # likely that someone will use a hyphen as a separator than
+ # a number, but at this point it's really best effort
+ # because this is an extension of the spec anyway.
+ # TODO(pganssle): Document this
+ return 8
+ return 10
+ else:
+ # YYYY-Www (8)
+ return 8
+ else:
+ # YYYY-MM-DD (10)
+ return 10
+ else:
+ if dtstr[4] == week_indicator:
+ # YYYYWww (7) or YYYYWwwd (8)
+ idx = 7
+ while idx < len_dtstr:
+ if not _is_ascii_digit(dtstr[idx]):
+ break
+ idx += 1
+
+ if idx < 9:
+ return idx
+
+ if idx % 2 == 0:
+ # If the index of the last number is even, it's YYYYWwwd
+ return 7
+ else:
+ return 8
+ else:
+ # YYYYMMDD (8)
+ return 8
+
+
def _parse_isoformat_date(dtstr):
- # It is assumed that this function will only be called with a
- # string of length exactly 10, and (though this is not used) ASCII-only
+ # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10,
+ # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator
+ assert len(dtstr) in (7, 8, 10)
year = int(dtstr[0:4])
- if dtstr[4] != '-':
- raise ValueError('Invalid date separator: %s' % dtstr[4])
+ has_sep = dtstr[4] == '-'
- month = int(dtstr[5:7])
+ pos = 4 + has_sep
+ if dtstr[pos:pos + 1] == "W":
+ # YYYY-?Www-?D?
+ pos += 1
+ weekno = int(dtstr[pos:pos + 2])
+ pos += 2
- if dtstr[7] != '-':
- raise ValueError('Invalid date separator')
+ dayno = 1
+ if len(dtstr) > pos:
+ if (dtstr[pos:pos + 1] == '-') != has_sep:
+ raise ValueError("Inconsistent use of dash separator")
- day = int(dtstr[8:10])
+ pos += has_sep
- return [year, month, day]
+ dayno = int(dtstr[pos:pos + 1])
+
+ return list(_isoweek_to_gregorian(year, weekno, dayno))
+ else:
+ month = int(dtstr[pos:pos + 2])
+ pos += 2
+ if (dtstr[pos:pos + 1] == "-") != has_sep:
+ raise ValueError("Inconsistent use of dash separator")
+
+ pos += has_sep
+ day = int(dtstr[pos:pos + 2])
+
+ return [year, month, day]
+
+
+_FRACTION_CORRECTION = [100000, 10000, 1000, 100, 10]
+
def _parse_hh_mm_ss_ff(tstr):
- # Parses things of the form HH[:MM[:SS[.fff[fff]]]]
+ # Parses things of the form HH[:?MM[:?SS[{.,}fff[fff]]]]
len_str = len(tstr)
time_comps = [0, 0, 0, 0]
pos = 0
for comp in range(0, 3):
if (len_str - pos) < 2:
- raise ValueError('Incomplete time component')
+ raise ValueError("Incomplete time component")
time_comps[comp] = int(tstr[pos:pos+2])
pos += 2
next_char = tstr[pos:pos+1]
+ if comp == 0:
+ has_sep = next_char == ':'
+
if not next_char or comp >= 2:
break
- if next_char != ':':
- raise ValueError('Invalid time separator: %c' % next_char)
+ if has_sep and next_char != ':':
+ raise ValueError("Invalid time separator: %c" % next_char)
- pos += 1
+ pos += has_sep
if pos < len_str:
- if tstr[pos] != '.':
- raise ValueError('Invalid microsecond component')
+ if tstr[pos] not in '.,':
+ raise ValueError("Invalid microsecond component")
else:
pos += 1
len_remainder = len_str - pos
- if len_remainder not in (3, 6):
- raise ValueError('Invalid microsecond component')
- time_comps[3] = int(tstr[pos:])
- if len_remainder == 3:
- time_comps[3] *= 1000
+ if len_remainder >= 6:
+ to_parse = 6
+ else:
+ to_parse = len_remainder
+
+ time_comps[3] = int(tstr[pos:(pos+to_parse)])
+ if to_parse < 6:
+ time_comps[3] *= _FRACTION_CORRECTION[to_parse-1]
+ if (len_remainder > to_parse
+ and not all(map(_is_ascii_digit, tstr[(pos+to_parse):]))):
+ raise ValueError("Non-digit values in unparsed fraction")
return time_comps
@@ -321,27 +413,34 @@
# Format supported is HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]
len_str = len(tstr)
if len_str < 2:
- raise ValueError('Isoformat time too short')
+ raise ValueError("Isoformat time too short")
- # This is equivalent to re.search('[+-]', tstr), but faster
- tz_pos = (tstr.find('-') + 1 or tstr.find('+') + 1)
+ # This is equivalent to re.search('[+-Z]', tstr), but faster
+ tz_pos = (tstr.find('-') + 1 or tstr.find('+') + 1 or tstr.find('Z') + 1)
timestr = tstr[:tz_pos-1] if tz_pos > 0 else tstr
time_comps = _parse_hh_mm_ss_ff(timestr)
tzi = None
- if tz_pos > 0:
+ if tz_pos == len_str and tstr[-1] == 'Z':
+ tzi = timezone.utc
+ elif tz_pos > 0:
tzstr = tstr[tz_pos:]
# Valid time zone strings are:
+ # HH len: 2
+ # HHMM len: 4
# HH:MM len: 5
+ # HHMMSS len: 6
+ # HHMMSS.f+ len: 7+
# HH:MM:SS len: 8
- # HH:MM:SS.ffffff len: 15
+ # HH:MM:SS.f+ len: 10+
- if len(tzstr) not in (5, 8, 15):
- raise ValueError('Malformed time zone string')
+ if len(tzstr) in (0, 1, 3):
+ raise ValueError("Malformed time zone string")
tz_comps = _parse_hh_mm_ss_ff(tzstr)
+
if all(x == 0 for x in tz_comps):
tzi = timezone.utc
else:
@@ -356,6 +455,38 @@
return time_comps
+# tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar
+def _isoweek_to_gregorian(year, week, day):
+ # Year is bounded this way because 9999-12-31 is (9999, 52, 5)
+ if not MINYEAR <= year <= MAXYEAR:
+ raise ValueError(f"Year is out of range: {year}")
+
+ if not 0 < week < 53:
+ out_of_range = True
+
+ if week == 53:
+ # ISO years have 53 weeks in them on years starting with a
+ # Thursday and leap years starting on a Wednesday
+ first_weekday = _ymd2ord(year, 1, 1) % 7
+ if (first_weekday == 4 or (first_weekday == 3 and
+ _is_leap(year))):
+ out_of_range = False
+
+ if out_of_range:
+ raise ValueError(f"Invalid week: {week}")
+
+ if not 0 < day < 8:
+ raise ValueError(f"Invalid weekday: {day} (range is [1, 7])")
+
+ # Now compute the offset from (Y, 1, 1) in days:
+ day_offset = (week - 1) * 7 + (day - 1)
+
+ # Calculate the ordinal day for monday, week 1
+ day_1 = _isoweek1monday(year)
+ ord_day = day_1 + day_offset
+
+ return _ord2ymd(ord_day)
+
# Just raise TypeError if the arg isn't None or a string.
def _check_tzname(name):
@@ -847,12 +978,14 @@
@classmethod
def fromisoformat(cls, date_string):
- """Construct a date from the output of date.isoformat()."""
+ """Construct a date from a string in ISO 8601 format."""
if not isinstance(date_string, str):
raise TypeError('fromisoformat: argument must be str')
+ if len(date_string) not in (7, 8, 10):
+ raise ValueError(f'Invalid isoformat string: {date_string!r}')
+
try:
- assert len(date_string) == 10
return cls(*_parse_isoformat_date(date_string))
except Exception:
raise ValueError(f'Invalid isoformat string: {date_string!r}')
@@ -862,35 +995,7 @@
"""Construct a date from the ISO year, week number and weekday.
This is the inverse of the date.isocalendar() function"""
- # Year is bounded this way because 9999-12-31 is (9999, 52, 5)
- if not MINYEAR <= year <= MAXYEAR:
- raise ValueError(f"Year is out of range: {year}")
-
- if not 0 < week < 53:
- out_of_range = True
-
- if week == 53:
- # ISO years have 53 weeks in them on years starting with a
- # Thursday and leap years starting on a Wednesday
- first_weekday = _ymd2ord(year, 1, 1) % 7
- if (first_weekday == 4 or (first_weekday == 3 and
- _is_leap(year))):
- out_of_range = False
-
- if out_of_range:
- raise ValueError(f"Invalid week: {week}")
-
- if not 0 < day < 8:
- raise ValueError(f"Invalid weekday: {day} (range is [1, 7])")
-
- # Now compute the offset from (Y, 1, 1) in days:
- day_offset = (week - 1) * 7 + (day - 1)
-
- # Calculate the ordinal day for monday, week 1
- day_1 = _isoweek1monday(year)
- ord_day = day_1 + day_offset
-
- return cls(*_ord2ymd(ord_day))
+ return cls(*_isoweek_to_gregorian(year, week, day))
# Conversions to string
@@ -925,7 +1030,11 @@
self._day, self._year)
def strftime(self, fmt):
- "Format using strftime()."
+ """
+ Format using strftime().
+
+ Example: "%d/%m/%Y, %H:%M:%S"
+ """
return _wrap_strftime(self, fmt, self.timetuple())
def __format__(self, fmt):
@@ -1169,15 +1278,7 @@
args = getinitargs()
else:
args = ()
- getstate = getattr(self, "__getstate__", None)
- if getstate:
- state = getstate()
- else:
- state = getattr(self, "__dict__", None) or None
- if state is None:
- return (self.__class__, args)
- else:
- return (self.__class__, args, state)
+ return (self.__class__, args, self.__getstate__())
class IsoCalendarDate(tuple):
@@ -1435,10 +1536,15 @@
@classmethod
def fromisoformat(cls, time_string):
- """Construct a time from the output of isoformat()."""
+ """Construct a time from a string in one of the ISO 8601 formats."""
if not isinstance(time_string, str):
raise TypeError('fromisoformat: argument must be str')
+ # The spec actually requires that time-only ISO 8601 strings start with
+ # T, but the extended format allows this to be omitted as long as there
+ # is no ambiguity with date strings.
+ time_string = time_string.removeprefix('T')
+
try:
return cls(*_parse_isoformat_time(time_string))
except Exception:
@@ -1719,24 +1825,30 @@
@classmethod
def fromisoformat(cls, date_string):
- """Construct a datetime from the output of datetime.isoformat()."""
+ """Construct a datetime from a string in one of the ISO 8601 formats."""
if not isinstance(date_string, str):
raise TypeError('fromisoformat: argument must be str')
- # Split this at the separator
- dstr = date_string[0:10]
- tstr = date_string[11:]
+ if len(date_string) < 7:
+ raise ValueError(f'Invalid isoformat string: {date_string!r}')
+ # Split this at the separator
try:
+ separator_location = _find_isoformat_datetime_separator(date_string)
+ dstr = date_string[0:separator_location]
+ tstr = date_string[(separator_location+1):]
+
date_components = _parse_isoformat_date(dstr)
except ValueError:
- raise ValueError(f'Invalid isoformat string: {date_string!r}')
+ raise ValueError(
+ f'Invalid isoformat string: {date_string!r}') from None
if tstr:
try:
time_components = _parse_isoformat_time(tstr)
except ValueError:
- raise ValueError(f'Invalid isoformat string: {date_string!r}')
+ raise ValueError(
+ f'Invalid isoformat string: {date_string!r}') from None
else:
time_components = [0, 0, 0, 0, None]
@@ -2298,7 +2410,8 @@
return f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}'
return f'UTC{sign}{hours:02d}:{minutes:02d}'
-timezone.utc = timezone._create(timedelta(0))
+UTC = timezone.utc = timezone._create(timedelta(0))
+
# bpo-37642: These attributes are rounded to the nearest minute for backwards
# compatibility, even though the constructor will accept a wider range of
# values. This may change in the future.
@@ -2516,7 +2629,9 @@
_format_time, _format_offset, _index, _is_leap, _isoweek1monday, _math,
_ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord,
_divide_and_round, _parse_isoformat_date, _parse_isoformat_time,
- _parse_hh_mm_ss_ff, _IsoCalendarDate)
+ _parse_hh_mm_ss_ff, _IsoCalendarDate, _isoweek_to_gregorian,
+ _find_isoformat_datetime_separator, _FRACTION_CORRECTION,
+ _is_ascii_digit)
# XXX Since import * above excludes names that start with _,
# docstring does not get overwritten. In the future, it may be
# appropriate to maintain a single module level docstring and
diff --git a/common/py3-stdlib/dis.py b/common/py3-stdlib/dis.py
index fe5d24e..5bf52c3 100644
--- a/common/py3-stdlib/dis.py
+++ b/common/py3-stdlib/dis.py
@@ -6,7 +6,14 @@
import io
from opcode import *
-from opcode import __all__ as _opcodes_all
+from opcode import (
+ __all__ as _opcodes_all,
+ _cache_format,
+ _inline_cache_entries,
+ _nb_ops,
+ _specializations,
+ _specialized_instructions,
+)
__all__ = ["code_info", "dis", "disassemble", "distb", "disco",
"findlinestarts", "findlabels", "show_code",
@@ -26,6 +33,24 @@
MAKE_FUNCTION = opmap['MAKE_FUNCTION']
MAKE_FUNCTION_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure')
+LOAD_CONST = opmap['LOAD_CONST']
+LOAD_GLOBAL = opmap['LOAD_GLOBAL']
+BINARY_OP = opmap['BINARY_OP']
+JUMP_BACKWARD = opmap['JUMP_BACKWARD']
+
+CACHE = opmap["CACHE"]
+
+_all_opname = list(opname)
+_all_opmap = dict(opmap)
+_empty_slot = [slot for slot, name in enumerate(_all_opname) if name.startswith("<")]
+for spec_op, specialized in zip(_empty_slot, _specialized_instructions):
+ # fill opname and opmap
+ _all_opname[spec_op] = specialized
+ _all_opmap[specialized] = spec_op
+
+deoptmap = {
+ specialized: base for base, family in _specializations.items() for specialized in family
+}
def _try_compile(source, name):
"""Attempts to compile the given source, first as an expression and
@@ -40,7 +65,7 @@
c = compile(source, name, 'exec')
return c
-def dis(x=None, *, file=None, depth=None):
+def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False):
"""Disassemble classes, methods, functions, and other compiled objects.
With no argument, disassemble the last traceback.
@@ -50,7 +75,7 @@
in a special attribute.
"""
if x is None:
- distb(file=file)
+ distb(file=file, show_caches=show_caches, adaptive=adaptive)
return
# Extract functions from methods.
if hasattr(x, '__func__'):
@@ -71,21 +96,21 @@
if isinstance(x1, _have_code):
print("Disassembly of %s:" % name, file=file)
try:
- dis(x1, file=file, depth=depth)
+ dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
except TypeError as msg:
print("Sorry:", msg, file=file)
print(file=file)
elif hasattr(x, 'co_code'): # Code object
- _disassemble_recursive(x, file=file, depth=depth)
+ _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
elif isinstance(x, (bytes, bytearray)): # Raw bytecode
- _disassemble_bytes(x, file=file)
+ _disassemble_bytes(x, file=file, show_caches=show_caches)
elif isinstance(x, str): # Source code
- _disassemble_str(x, file=file, depth=depth)
+ _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
else:
raise TypeError("don't know how to disassemble %s objects" %
type(x).__name__)
-def distb(tb=None, *, file=None):
+def distb(tb=None, *, file=None, show_caches=False, adaptive=False):
"""Disassemble a traceback (default: last traceback)."""
if tb is None:
try:
@@ -93,7 +118,7 @@
except AttributeError:
raise RuntimeError("no last traceback to disassemble") from None
while tb.tb_next: tb = tb.tb_next
- disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file)
+ disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive)
# The inspect module interrogates this dictionary to build its
# list of CO_* constants. It is also used by pretty_flags to
@@ -125,6 +150,13 @@
names.append(hex(flags))
return ", ".join(names)
+class _Unknown:
+ def __repr__(self):
+ return "<unknown>"
+
+# Sentinel to represent values that cannot be calculated
+UNKNOWN = _Unknown()
+
def _get_code_object(x):
"""Helper to handle methods, compiled or raw code objects, and strings."""
# Extract functions from methods.
@@ -148,6 +180,13 @@
raise TypeError("don't know how to disassemble %s objects" %
type(x).__name__)
+def _deoptop(op):
+ name = _all_opname[op]
+ return _all_opmap[deoptmap[name]] if name in deoptmap else op
+
+def _get_code_array(co, adaptive):
+ return co._co_code_adaptive if adaptive else co.co_code
+
def code_info(x):
"""Formatted details of methods, functions, or code."""
return _format_code_info(_get_code_object(x))
@@ -191,8 +230,32 @@
"""
print(code_info(co), file=file)
-_Instruction = collections.namedtuple("_Instruction",
- "opname opcode arg argval argrepr offset starts_line is_jump_target")
+Positions = collections.namedtuple(
+ 'Positions',
+ [
+ 'lineno',
+ 'end_lineno',
+ 'col_offset',
+ 'end_col_offset',
+ ],
+ defaults=[None] * 4
+)
+
+_Instruction = collections.namedtuple(
+ "_Instruction",
+ [
+ 'opname',
+ 'opcode',
+ 'arg',
+ 'argval',
+ 'argrepr',
+ 'offset',
+ 'starts_line',
+ 'is_jump_target',
+ 'positions'
+ ],
+ defaults=[None]
+)
_Instruction.opname.__doc__ = "Human readable name for operation"
_Instruction.opcode.__doc__ = "Numeric code for operation"
@@ -202,6 +265,10 @@
_Instruction.offset.__doc__ = "Start index of operation within bytecode sequence"
_Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None"
_Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False"
+_Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction"
+
+_ExceptionTableEntry = collections.namedtuple("_ExceptionTableEntry",
+ "start end target depth lasti")
_OPNAME_WIDTH = 20
_OPARG_WIDTH = 5
@@ -218,6 +285,8 @@
offset - start index of operation within bytecode sequence
starts_line - line started by this opcode (if any), otherwise None
is_jump_target - True if other code jumps to here, otherwise False
+ positions - Optional dis.Positions object holding the span of source code
+ covered by this instruction
"""
def _disassemble(self, lineno_width=3, mark_as_current=False, offset_width=4):
@@ -258,7 +327,7 @@
return ' '.join(fields).rstrip()
-def get_instructions(x, *, first_line=None):
+def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False):
"""Iterator for the opcodes in methods, functions or code
Generates a series of Instruction named tuples giving the details of
@@ -270,55 +339,106 @@
the disassembled code object.
"""
co = _get_code_object(x)
- cell_names = co.co_cellvars + co.co_freevars
linestarts = dict(findlinestarts(co))
if first_line is not None:
line_offset = first_line - co.co_firstlineno
else:
line_offset = 0
- return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
- co.co_consts, cell_names, linestarts,
- line_offset)
+ return _get_instructions_bytes(_get_code_array(co, adaptive),
+ co._varname_from_oparg,
+ co.co_names, co.co_consts,
+ linestarts, line_offset,
+ co_positions=co.co_positions(),
+ show_caches=show_caches)
-def _get_const_info(const_index, const_list):
+def _get_const_value(op, arg, co_consts):
+ """Helper to get the value of the const in a hasconst op.
+
+ Returns the dereferenced constant if this is possible.
+ Otherwise (if it is a LOAD_CONST and co_consts is not
+ provided) returns the dis.UNKNOWN sentinel.
+ """
+ assert op in hasconst
+
+ argval = UNKNOWN
+ if op == LOAD_CONST:
+ if co_consts is not None:
+ argval = co_consts[arg]
+ return argval
+
+def _get_const_info(op, arg, co_consts):
"""Helper to get optional details about const references
- Returns the dereferenced constant and its repr if the constant
- list is defined.
- Otherwise returns the constant index and its repr().
+ Returns the dereferenced constant and its repr if the value
+ can be calculated.
+ Otherwise returns the sentinel value dis.UNKNOWN for the value
+ and an empty string for its repr.
"""
- argval = const_index
- if const_list is not None:
- argval = const_list[const_index]
- return argval, repr(argval)
+ argval = _get_const_value(op, arg, co_consts)
+ argrepr = repr(argval) if argval is not UNKNOWN else ''
+ return argval, argrepr
-def _get_name_info(name_index, name_list):
+def _get_name_info(name_index, get_name, **extrainfo):
"""Helper to get optional details about named references
Returns the dereferenced name as both value and repr if the name
list is defined.
- Otherwise returns the name index and its repr().
+ Otherwise returns the sentinel value dis.UNKNOWN for the value
+ and an empty string for its repr.
"""
- argval = name_index
- if name_list is not None:
- argval = name_list[name_index]
- argrepr = argval
+ if get_name is not None:
+ argval = get_name(name_index, **extrainfo)
+ return argval, argval
else:
- argrepr = repr(argval)
- return argval, argrepr
+ return UNKNOWN, ''
+def _parse_varint(iterator):
+ b = next(iterator)
+ val = b & 63
+ while b&64:
+ val <<= 6
+ b = next(iterator)
+ val |= b&63
+ return val
-def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
- cells=None, linestarts=None, line_offset=0):
+def _parse_exception_table(code):
+ iterator = iter(code.co_exceptiontable)
+ entries = []
+ try:
+ while True:
+ start = _parse_varint(iterator)*2
+ length = _parse_varint(iterator)*2
+ end = start + length
+ target = _parse_varint(iterator)*2
+ dl = _parse_varint(iterator)
+ depth = dl >> 1
+ lasti = bool(dl&1)
+ entries.append(_ExceptionTableEntry(start, end, target, depth, lasti))
+ except StopIteration:
+ return entries
+
+def _is_backward_jump(op):
+ return 'JUMP_BACKWARD' in opname[op]
+
+def _get_instructions_bytes(code, varname_from_oparg=None,
+ names=None, co_consts=None,
+ linestarts=None, line_offset=0,
+ exception_entries=(), co_positions=None,
+ show_caches=False):
"""Iterate over the instructions in a bytecode string.
Generates a sequence of Instruction namedtuples giving the details of each
opcode. Additional information about the code's runtime environment
- (e.g. variable names, constants) can be specified using optional
+ (e.g. variable names, co_consts) can be specified using optional
arguments.
"""
- labels = findlabels(code)
+ co_positions = co_positions or iter(())
+ get_name = None if names is None else names.__getitem__
+ labels = set(findlabels(code))
+ for start, end, target, _, _ in exception_entries:
+ for i in range(start, end):
+ labels.add(target)
starts_line = None
for offset, op, arg in _unpack_opargs(code):
if linestarts is not None:
@@ -328,52 +448,86 @@
is_jump_target = offset in labels
argval = None
argrepr = ''
+ positions = Positions(*next(co_positions, ()))
+ deop = _deoptop(op)
if arg is not None:
# Set argval to the dereferenced value of the argument when
# available, and argrepr to the string representation of argval.
# _disassemble_bytes needs the string repr of the
# raw name index for LOAD_GLOBAL, LOAD_CONST, etc.
argval = arg
- if op in hasconst:
- argval, argrepr = _get_const_info(arg, constants)
- elif op in hasname:
- argval, argrepr = _get_name_info(arg, names)
- elif op in hasjabs:
+ if deop in hasconst:
+ argval, argrepr = _get_const_info(deop, arg, co_consts)
+ elif deop in hasname:
+ if deop == LOAD_GLOBAL:
+ argval, argrepr = _get_name_info(arg//2, get_name)
+ if (arg & 1) and argrepr:
+ argrepr = "NULL + " + argrepr
+ else:
+ argval, argrepr = _get_name_info(arg, get_name)
+ elif deop in hasjabs:
argval = arg*2
argrepr = "to " + repr(argval)
- elif op in hasjrel:
- argval = offset + 2 + arg*2
+ elif deop in hasjrel:
+ signed_arg = -arg if _is_backward_jump(deop) else arg
+ argval = offset + 2 + signed_arg*2
argrepr = "to " + repr(argval)
- elif op in haslocal:
- argval, argrepr = _get_name_info(arg, varnames)
- elif op in hascompare:
+ elif deop in haslocal or deop in hasfree:
+ argval, argrepr = _get_name_info(arg, varname_from_oparg)
+ elif deop in hascompare:
argval = cmp_op[arg]
argrepr = argval
- elif op in hasfree:
- argval, argrepr = _get_name_info(arg, cells)
- elif op == FORMAT_VALUE:
+ elif deop == FORMAT_VALUE:
argval, argrepr = FORMAT_VALUE_CONVERTERS[arg & 0x3]
argval = (argval, bool(arg & 0x4))
if argval[1]:
if argrepr:
argrepr += ', '
argrepr += 'with format'
- elif op == MAKE_FUNCTION:
+ elif deop == MAKE_FUNCTION:
argrepr = ', '.join(s for i, s in enumerate(MAKE_FUNCTION_FLAGS)
if arg & (1<<i))
- yield Instruction(opname[op], op,
+ elif deop == BINARY_OP:
+ _, argrepr = _nb_ops[arg]
+ yield Instruction(_all_opname[op], op,
arg, argval, argrepr,
- offset, starts_line, is_jump_target)
+ offset, starts_line, is_jump_target, positions)
+ caches = _inline_cache_entries[deop]
+ if not caches:
+ continue
+ if not show_caches:
+ # We still need to advance the co_positions iterator:
+ for _ in range(caches):
+ next(co_positions, ())
+ continue
+ for name, size in _cache_format[opname[deop]].items():
+ for i in range(size):
+ offset += 2
+ # Only show the fancy argrepr for a CACHE instruction when it's
+ # the first entry for a particular cache value and the
+ # instruction using it is actually quickened:
+ if i == 0 and op != deop:
+ data = code[offset: offset + 2 * size]
+ argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}"
+ else:
+ argrepr = ""
+ yield Instruction(
+ "CACHE", CACHE, 0, None, argrepr, offset, None, False,
+ Positions(*next(co_positions, ()))
+ )
-def disassemble(co, lasti=-1, *, file=None):
+def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False):
"""Disassemble a code object."""
- cell_names = co.co_cellvars + co.co_freevars
linestarts = dict(findlinestarts(co))
- _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names,
- co.co_consts, cell_names, linestarts, file=file)
+ exception_entries = _parse_exception_table(co)
+ _disassemble_bytes(_get_code_array(co, adaptive),
+ lasti, co._varname_from_oparg,
+ co.co_names, co.co_consts, linestarts, file=file,
+ exception_entries=exception_entries,
+ co_positions=co.co_positions(), show_caches=show_caches)
-def _disassemble_recursive(co, *, file=None, depth=None):
- disassemble(co, file=file)
+def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False):
+ disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive)
if depth is None or depth > 0:
if depth is not None:
depth = depth - 1
@@ -381,11 +535,14 @@
if hasattr(x, 'co_code'):
print(file=file)
print("Disassembly of %r:" % (x,), file=file)
- _disassemble_recursive(x, file=file, depth=depth)
+ _disassemble_recursive(
+ x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive
+ )
-def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
- constants=None, cells=None, linestarts=None,
- *, file=None, line_offset=0):
+def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None,
+ names=None, co_consts=None, linestarts=None,
+ *, file=None, line_offset=0, exception_entries=(),
+ co_positions=None, show_caches=False):
# Omit the line number column entirely if we have no line number info
show_lineno = bool(linestarts)
if show_lineno:
@@ -401,9 +558,12 @@
offset_width = len(str(maxoffset))
else:
offset_width = 4
- for instr in _get_instructions_bytes(code, varnames, names,
- constants, cells, linestarts,
- line_offset=line_offset):
+ for instr in _get_instructions_bytes(code, varname_from_oparg, names,
+ co_consts, linestarts,
+ line_offset=line_offset,
+ exception_entries=exception_entries,
+ co_positions=co_positions,
+ show_caches=show_caches):
new_source_line = (show_lineno and
instr.starts_line is not None and
instr.offset > 0)
@@ -412,6 +572,12 @@
is_current_instr = instr.offset == lasti
print(instr._disassemble(lineno_width, is_current_instr, offset_width),
file=file)
+ if exception_entries:
+ print("ExceptionTable:", file=file)
+ for entry in exception_entries:
+ lasti = " lasti" if entry.lasti else ""
+ end = entry.end-2
+ print(f" {entry.start} to {end} -> {entry.target} [{entry.depth}]{lasti}", file=file)
def _disassemble_str(source, **kwargs):
"""Compile the source string, then disassemble the code object."""
@@ -419,13 +585,31 @@
disco = disassemble # XXX For backwards compatibility
+
+# Rely on C `int` being 32 bits for oparg
+_INT_BITS = 32
+# Value for c int when it overflows
+_INT_OVERFLOW = 2 ** (_INT_BITS - 1)
+
def _unpack_opargs(code):
extended_arg = 0
+ caches = 0
for i in range(0, len(code), 2):
+ # Skip inline CACHE entries:
+ if caches:
+ caches -= 1
+ continue
op = code[i]
- if op >= HAVE_ARGUMENT:
+ deop = _deoptop(op)
+ caches = _inline_cache_entries[deop]
+ if deop >= HAVE_ARGUMENT:
arg = code[i+1] | extended_arg
- extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
+ extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0
+ # The oparg is stored as a signed integer
+ # If the value exceeds its upper limit, it will overflow and wrap
+ # to a negative integer
+ if extended_arg >= _INT_OVERFLOW:
+ extended_arg -= 2 * _INT_OVERFLOW
else:
arg = None
extended_arg = 0
@@ -441,6 +625,8 @@
for offset, op, arg in _unpack_opargs(code):
if arg is not None:
if op in hasjrel:
+ if _is_backward_jump(op):
+ arg = -arg
label = offset + 2 + arg*2
elif op in hasjabs:
label = arg*2
@@ -462,6 +648,44 @@
yield start, line
return
+def _find_imports(co):
+ """Find import statements in the code
+
+ Generate triplets (name, level, fromlist) where
+ name is the imported module and level, fromlist are
+ the corresponding args to __import__.
+ """
+ IMPORT_NAME = opmap['IMPORT_NAME']
+ LOAD_CONST = opmap['LOAD_CONST']
+
+ consts = co.co_consts
+ names = co.co_names
+ opargs = [(op, arg) for _, op, arg in _unpack_opargs(co.co_code)
+ if op != EXTENDED_ARG]
+ for i, (op, oparg) in enumerate(opargs):
+ if op == IMPORT_NAME and i >= 2:
+ from_op = opargs[i-1]
+ level_op = opargs[i-2]
+ if (from_op[0] in hasconst and level_op[0] in hasconst):
+ level = _get_const_value(level_op[0], level_op[1], consts)
+ fromlist = _get_const_value(from_op[0], from_op[1], consts)
+ yield (names[oparg], level, fromlist)
+
+def _find_store_names(co):
+ """Find names of variables which are written in the code
+
+ Generate sequence of strings
+ """
+ STORE_OPS = {
+ opmap['STORE_NAME'],
+ opmap['STORE_GLOBAL']
+ }
+
+ names = co.co_names
+ for _, op, arg in _unpack_opargs(co.co_code):
+ if op in STORE_OPS:
+ yield names[arg]
+
class Bytecode:
"""The bytecode operations of a piece of code
@@ -471,7 +695,7 @@
Iterating over this yields the bytecode operations as Instruction instances.
"""
- def __init__(self, x, *, first_line=None, current_offset=None):
+ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False):
self.codeobj = co = _get_code_object(x)
if first_line is None:
self.first_line = co.co_firstlineno
@@ -479,28 +703,36 @@
else:
self.first_line = first_line
self._line_offset = first_line - co.co_firstlineno
- self._cell_names = co.co_cellvars + co.co_freevars
self._linestarts = dict(findlinestarts(co))
self._original_object = x
self.current_offset = current_offset
+ self.exception_entries = _parse_exception_table(co)
+ self.show_caches = show_caches
+ self.adaptive = adaptive
def __iter__(self):
co = self.codeobj
- return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
- co.co_consts, self._cell_names,
+ return _get_instructions_bytes(_get_code_array(co, self.adaptive),
+ co._varname_from_oparg,
+ co.co_names, co.co_consts,
self._linestarts,
- line_offset=self._line_offset)
+ line_offset=self._line_offset,
+ exception_entries=self.exception_entries,
+ co_positions=co.co_positions(),
+ show_caches=self.show_caches)
def __repr__(self):
return "{}({!r})".format(self.__class__.__name__,
self._original_object)
@classmethod
- def from_traceback(cls, tb):
+ def from_traceback(cls, tb, *, show_caches=False, adaptive=False):
""" Construct a Bytecode from the given traceback """
while tb.tb_next:
tb = tb.tb_next
- return cls(tb.tb_frame.f_code, current_offset=tb.tb_lasti)
+ return cls(
+ tb.tb_frame.f_code, current_offset=tb.tb_lasti, show_caches=show_caches, adaptive=adaptive
+ )
def info(self):
"""Return formatted information about the code object."""
@@ -514,13 +746,16 @@
else:
offset = -1
with io.StringIO() as output:
- _disassemble_bytes(co.co_code, varnames=co.co_varnames,
- names=co.co_names, constants=co.co_consts,
- cells=self._cell_names,
+ _disassemble_bytes(_get_code_array(co, self.adaptive),
+ varname_from_oparg=co._varname_from_oparg,
+ names=co.co_names, co_consts=co.co_consts,
linestarts=self._linestarts,
line_offset=self._line_offset,
file=output,
- lasti=offset)
+ lasti=offset,
+ exception_entries=self.exception_entries,
+ co_positions=co.co_positions(),
+ show_caches=self.show_caches)
return output.getvalue()
diff --git a/common/py3-stdlib/distutils/command/bdist.py b/common/py3-stdlib/distutils/command/bdist.py
index d580a80..60309e1 100644
--- a/common/py3-stdlib/distutils/command/bdist.py
+++ b/common/py3-stdlib/distutils/command/bdist.py
@@ -61,8 +61,7 @@
'nt': 'zip'}
# Establish the preferred order (for the --help-formats option).
- format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar',
- 'zip', 'msi']
+ format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', 'zip']
# And the real information.
format_command = {'rpm': ('bdist_rpm', "RPM distribution"),
@@ -72,10 +71,8 @@
'ztar': ('bdist_dumb', "compressed tar file"),
'tar': ('bdist_dumb', "tar file"),
'zip': ('bdist_dumb', "ZIP file"),
- 'msi': ('bdist_msi', "Microsoft Installer")
}
-
def initialize_options(self):
self.bdist_base = None
self.plat_name = None
diff --git a/common/py3-stdlib/distutils/command/bdist_msi.py b/common/py3-stdlib/distutils/command/bdist_msi.py
deleted file mode 100644
index 2ed017b..0000000
--- a/common/py3-stdlib/distutils/command/bdist_msi.py
+++ /dev/null
@@ -1,747 +0,0 @@
-# Copyright (C) 2005, 2006 Martin von Löwis
-# Licensed to PSF under a Contributor Agreement.
-"""
-Implements the bdist_msi command.
-"""
-
-import os
-import sys
-import warnings
-from distutils.core import Command
-from distutils.dir_util import remove_tree
-from distutils.sysconfig import get_python_version
-from distutils.version import StrictVersion
-from distutils.errors import DistutilsOptionError
-from distutils.util import get_platform
-from distutils import log
-import msilib
-from msilib import schema, sequence, text
-from msilib import Directory, Feature, Dialog, add_data
-
-class PyDialog(Dialog):
- """Dialog class with a fixed layout: controls at the top, then a ruler,
- then a list of buttons: back, next, cancel. Optionally a bitmap at the
- left."""
- def __init__(self, *args, **kw):
- """Dialog(database, name, x, y, w, h, attributes, title, first,
- default, cancel, bitmap=true)"""
- Dialog.__init__(self, *args)
- ruler = self.h - 36
- bmwidth = 152*ruler/328
- #if kw.get("bitmap", True):
- # self.bitmap("Bitmap", 0, 0, bmwidth, ruler, "PythonWin")
- self.line("BottomLine", 0, ruler, self.w, 0)
-
- def title(self, title):
- "Set the title text of the dialog at the top."
- # name, x, y, w, h, flags=Visible|Enabled|Transparent|NoPrefix,
- # text, in VerdanaBold10
- self.text("Title", 15, 10, 320, 60, 0x30003,
- r"{\VerdanaBold10}%s" % title)
-
- def back(self, title, next, name = "Back", active = 1):
- """Add a back button with a given title, the tab-next button,
- its name in the Control table, possibly initially disabled.
-
- Return the button, so that events can be associated"""
- if active:
- flags = 3 # Visible|Enabled
- else:
- flags = 1 # Visible
- return self.pushbutton(name, 180, self.h-27 , 56, 17, flags, title, next)
-
- def cancel(self, title, next, name = "Cancel", active = 1):
- """Add a cancel button with a given title, the tab-next button,
- its name in the Control table, possibly initially disabled.
-
- Return the button, so that events can be associated"""
- if active:
- flags = 3 # Visible|Enabled
- else:
- flags = 1 # Visible
- return self.pushbutton(name, 304, self.h-27, 56, 17, flags, title, next)
-
- def next(self, title, next, name = "Next", active = 1):
- """Add a Next button with a given title, the tab-next button,
- its name in the Control table, possibly initially disabled.
-
- Return the button, so that events can be associated"""
- if active:
- flags = 3 # Visible|Enabled
- else:
- flags = 1 # Visible
- return self.pushbutton(name, 236, self.h-27, 56, 17, flags, title, next)
-
- def xbutton(self, name, title, next, xpos):
- """Add a button with a given title, the tab-next button,
- its name in the Control table, giving its x position; the
- y-position is aligned with the other buttons.
-
- Return the button, so that events can be associated"""
- return self.pushbutton(name, int(self.w*xpos - 28), self.h-27, 56, 17, 3, title, next)
-
-class bdist_msi(Command):
-
- description = "create a Microsoft Installer (.msi) binary distribution"
-
- user_options = [('bdist-dir=', None,
- "temporary directory for creating the distribution"),
- ('plat-name=', 'p',
- "platform name to embed in generated filenames "
- "(default: %s)" % get_platform()),
- ('keep-temp', 'k',
- "keep the pseudo-installation tree around after " +
- "creating the distribution archive"),
- ('target-version=', None,
- "require a specific python version" +
- " on the target system"),
- ('no-target-compile', 'c',
- "do not compile .py to .pyc on the target system"),
- ('no-target-optimize', 'o',
- "do not compile .py to .pyo (optimized) "
- "on the target system"),
- ('dist-dir=', 'd',
- "directory to put final built distributions in"),
- ('skip-build', None,
- "skip rebuilding everything (for testing/debugging)"),
- ('install-script=', None,
- "basename of installation script to be run after "
- "installation or before deinstallation"),
- ('pre-install-script=', None,
- "Fully qualified filename of a script to be run before "
- "any files are installed. This script need not be in the "
- "distribution"),
- ]
-
- boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize',
- 'skip-build']
-
- all_versions = ['2.0', '2.1', '2.2', '2.3', '2.4',
- '2.5', '2.6', '2.7', '2.8', '2.9',
- '3.0', '3.1', '3.2', '3.3', '3.4',
- '3.5', '3.6', '3.7', '3.8', '3.9']
- other_version = 'X'
-
- def __init__(self, *args, **kw):
- super().__init__(*args, **kw)
- warnings.warn("bdist_msi command is deprecated since Python 3.9, "
- "use bdist_wheel (wheel packages) instead",
- DeprecationWarning, 2)
-
- def initialize_options(self):
- self.bdist_dir = None
- self.plat_name = None
- self.keep_temp = 0
- self.no_target_compile = 0
- self.no_target_optimize = 0
- self.target_version = None
- self.dist_dir = None
- self.skip_build = None
- self.install_script = None
- self.pre_install_script = None
- self.versions = None
-
- def finalize_options(self):
- self.set_undefined_options('bdist', ('skip_build', 'skip_build'))
-
- if self.bdist_dir is None:
- bdist_base = self.get_finalized_command('bdist').bdist_base
- self.bdist_dir = os.path.join(bdist_base, 'msi')
-
- short_version = get_python_version()
- if (not self.target_version) and self.distribution.has_ext_modules():
- self.target_version = short_version
-
- if self.target_version:
- self.versions = [self.target_version]
- if not self.skip_build and self.distribution.has_ext_modules()\
- and self.target_version != short_version:
- raise DistutilsOptionError(
- "target version can only be %s, or the '--skip-build'"
- " option must be specified" % (short_version,))
- else:
- self.versions = list(self.all_versions)
-
- self.set_undefined_options('bdist',
- ('dist_dir', 'dist_dir'),
- ('plat_name', 'plat_name'),
- )
-
- if self.pre_install_script:
- raise DistutilsOptionError(
- "the pre-install-script feature is not yet implemented")
-
- if self.install_script:
- for script in self.distribution.scripts:
- if self.install_script == os.path.basename(script):
- break
- else:
- raise DistutilsOptionError(
- "install_script '%s' not found in scripts"
- % self.install_script)
- self.install_script_key = None
-
- def run(self):
- if not self.skip_build:
- self.run_command('build')
-
- install = self.reinitialize_command('install', reinit_subcommands=1)
- install.prefix = self.bdist_dir
- install.skip_build = self.skip_build
- install.warn_dir = 0
-
- install_lib = self.reinitialize_command('install_lib')
- # we do not want to include pyc or pyo files
- install_lib.compile = 0
- install_lib.optimize = 0
-
- if self.distribution.has_ext_modules():
- # If we are building an installer for a Python version other
- # than the one we are currently running, then we need to ensure
- # our build_lib reflects the other Python version rather than ours.
- # Note that for target_version!=sys.version, we must have skipped the
- # build step, so there is no issue with enforcing the build of this
- # version.
- target_version = self.target_version
- if not target_version:
- assert self.skip_build, "Should have already checked this"
- target_version = '%d.%d' % sys.version_info[:2]
- plat_specifier = ".%s-%s" % (self.plat_name, target_version)
- build = self.get_finalized_command('build')
- build.build_lib = os.path.join(build.build_base,
- 'lib' + plat_specifier)
-
- log.info("installing to %s", self.bdist_dir)
- install.ensure_finalized()
-
- # avoid warning of 'install_lib' about installing
- # into a directory not in sys.path
- sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))
-
- install.run()
-
- del sys.path[0]
-
- self.mkpath(self.dist_dir)
- fullname = self.distribution.get_fullname()
- installer_name = self.get_installer_filename(fullname)
- installer_name = os.path.abspath(installer_name)
- if os.path.exists(installer_name): os.unlink(installer_name)
-
- metadata = self.distribution.metadata
- author = metadata.author
- if not author:
- author = metadata.maintainer
- if not author:
- author = "UNKNOWN"
- version = metadata.get_version()
- # ProductVersion must be strictly numeric
- # XXX need to deal with prerelease versions
- sversion = "%d.%d.%d" % StrictVersion(version).version
- # Prefix ProductName with Python x.y, so that
- # it sorts together with the other Python packages
- # in Add-Remove-Programs (APR)
- fullname = self.distribution.get_fullname()
- if self.target_version:
- product_name = "Python %s %s" % (self.target_version, fullname)
- else:
- product_name = "Python %s" % (fullname)
- self.db = msilib.init_database(installer_name, schema,
- product_name, msilib.gen_uuid(),
- sversion, author)
- msilib.add_tables(self.db, sequence)
- props = [('DistVersion', version)]
- email = metadata.author_email or metadata.maintainer_email
- if email:
- props.append(("ARPCONTACT", email))
- if metadata.url:
- props.append(("ARPURLINFOABOUT", metadata.url))
- if props:
- add_data(self.db, 'Property', props)
-
- self.add_find_python()
- self.add_files()
- self.add_scripts()
- self.add_ui()
- self.db.Commit()
-
- if hasattr(self.distribution, 'dist_files'):
- tup = 'bdist_msi', self.target_version or 'any', fullname
- self.distribution.dist_files.append(tup)
-
- if not self.keep_temp:
- remove_tree(self.bdist_dir, dry_run=self.dry_run)
-
- def add_files(self):
- db = self.db
- cab = msilib.CAB("distfiles")
- rootdir = os.path.abspath(self.bdist_dir)
-
- root = Directory(db, cab, None, rootdir, "TARGETDIR", "SourceDir")
- f = Feature(db, "Python", "Python", "Everything",
- 0, 1, directory="TARGETDIR")
-
- items = [(f, root, '')]
- for version in self.versions + [self.other_version]:
- target = "TARGETDIR" + version
- name = default = "Python" + version
- desc = "Everything"
- if version is self.other_version:
- title = "Python from another location"
- level = 2
- else:
- title = "Python %s from registry" % version
- level = 1
- f = Feature(db, name, title, desc, 1, level, directory=target)
- dir = Directory(db, cab, root, rootdir, target, default)
- items.append((f, dir, version))
- db.Commit()
-
- seen = {}
- for feature, dir, version in items:
- todo = [dir]
- while todo:
- dir = todo.pop()
- for file in os.listdir(dir.absolute):
- afile = os.path.join(dir.absolute, file)
- if os.path.isdir(afile):
- short = "%s|%s" % (dir.make_short(file), file)
- default = file + version
- newdir = Directory(db, cab, dir, file, default, short)
- todo.append(newdir)
- else:
- if not dir.component:
- dir.start_component(dir.logical, feature, 0)
- if afile not in seen:
- key = seen[afile] = dir.add_file(file)
- if file==self.install_script:
- if self.install_script_key:
- raise DistutilsOptionError(
- "Multiple files with name %s" % file)
- self.install_script_key = '[#%s]' % key
- else:
- key = seen[afile]
- add_data(self.db, "DuplicateFile",
- [(key + version, dir.component, key, None, dir.logical)])
- db.Commit()
- cab.commit(db)
-
- def add_find_python(self):
- """Adds code to the installer to compute the location of Python.
-
- Properties PYTHON.MACHINE.X.Y and PYTHON.USER.X.Y will be set from the
- registry for each version of Python.
-
- Properties TARGETDIRX.Y will be set from PYTHON.USER.X.Y if defined,
- else from PYTHON.MACHINE.X.Y.
-
- Properties PYTHONX.Y will be set to TARGETDIRX.Y\\python.exe"""
-
- start = 402
- for ver in self.versions:
- install_path = r"SOFTWARE\Python\PythonCore\%s\InstallPath" % ver
- machine_reg = "python.machine." + ver
- user_reg = "python.user." + ver
- machine_prop = "PYTHON.MACHINE." + ver
- user_prop = "PYTHON.USER." + ver
- machine_action = "PythonFromMachine" + ver
- user_action = "PythonFromUser" + ver
- exe_action = "PythonExe" + ver
- target_dir_prop = "TARGETDIR" + ver
- exe_prop = "PYTHON" + ver
- if msilib.Win64:
- # type: msidbLocatorTypeRawValue + msidbLocatorType64bit
- Type = 2+16
- else:
- Type = 2
- add_data(self.db, "RegLocator",
- [(machine_reg, 2, install_path, None, Type),
- (user_reg, 1, install_path, None, Type)])
- add_data(self.db, "AppSearch",
- [(machine_prop, machine_reg),
- (user_prop, user_reg)])
- add_data(self.db, "CustomAction",
- [(machine_action, 51+256, target_dir_prop, "[" + machine_prop + "]"),
- (user_action, 51+256, target_dir_prop, "[" + user_prop + "]"),
- (exe_action, 51+256, exe_prop, "[" + target_dir_prop + "]\\python.exe"),
- ])
- add_data(self.db, "InstallExecuteSequence",
- [(machine_action, machine_prop, start),
- (user_action, user_prop, start + 1),
- (exe_action, None, start + 2),
- ])
- add_data(self.db, "InstallUISequence",
- [(machine_action, machine_prop, start),
- (user_action, user_prop, start + 1),
- (exe_action, None, start + 2),
- ])
- add_data(self.db, "Condition",
- [("Python" + ver, 0, "NOT TARGETDIR" + ver)])
- start += 4
- assert start < 500
-
- def add_scripts(self):
- if self.install_script:
- start = 6800
- for ver in self.versions + [self.other_version]:
- install_action = "install_script." + ver
- exe_prop = "PYTHON" + ver
- add_data(self.db, "CustomAction",
- [(install_action, 50, exe_prop, self.install_script_key)])
- add_data(self.db, "InstallExecuteSequence",
- [(install_action, "&Python%s=3" % ver, start)])
- start += 1
- # XXX pre-install scripts are currently refused in finalize_options()
- # but if this feature is completed, it will also need to add
- # entries for each version as the above code does
- if self.pre_install_script:
- scriptfn = os.path.join(self.bdist_dir, "preinstall.bat")
- with open(scriptfn, "w") as f:
- # The batch file will be executed with [PYTHON], so that %1
- # is the path to the Python interpreter; %0 will be the path
- # of the batch file.
- # rem ="""
- # %1 %0
- # exit
- # """
- # <actual script>
- f.write('rem ="""\n%1 %0\nexit\n"""\n')
- with open(self.pre_install_script) as fin:
- f.write(fin.read())
- add_data(self.db, "Binary",
- [("PreInstall", msilib.Binary(scriptfn))
- ])
- add_data(self.db, "CustomAction",
- [("PreInstall", 2, "PreInstall", None)
- ])
- add_data(self.db, "InstallExecuteSequence",
- [("PreInstall", "NOT Installed", 450)])
-
-
- def add_ui(self):
- db = self.db
- x = y = 50
- w = 370
- h = 300
- title = "[ProductName] Setup"
-
- # see "Dialog Style Bits"
- modal = 3 # visible | modal
- modeless = 1 # visible
- track_disk_space = 32
-
- # UI customization properties
- add_data(db, "Property",
- # See "DefaultUIFont Property"
- [("DefaultUIFont", "DlgFont8"),
- # See "ErrorDialog Style Bit"
- ("ErrorDialog", "ErrorDlg"),
- ("Progress1", "Install"), # modified in maintenance type dlg
- ("Progress2", "installs"),
- ("MaintenanceForm_Action", "Repair"),
- # possible values: ALL, JUSTME
- ("WhichUsers", "ALL")
- ])
-
- # Fonts, see "TextStyle Table"
- add_data(db, "TextStyle",
- [("DlgFont8", "Tahoma", 9, None, 0),
- ("DlgFontBold8", "Tahoma", 8, None, 1), #bold
- ("VerdanaBold10", "Verdana", 10, None, 1),
- ("VerdanaRed9", "Verdana", 9, 255, 0),
- ])
-
- # UI Sequences, see "InstallUISequence Table", "Using a Sequence Table"
- # Numbers indicate sequence; see sequence.py for how these action integrate
- add_data(db, "InstallUISequence",
- [("PrepareDlg", "Not Privileged or Windows9x or Installed", 140),
- ("WhichUsersDlg", "Privileged and not Windows9x and not Installed", 141),
- # In the user interface, assume all-users installation if privileged.
- ("SelectFeaturesDlg", "Not Installed", 1230),
- # XXX no support for resume installations yet
- #("ResumeDlg", "Installed AND (RESUME OR Preselected)", 1240),
- ("MaintenanceTypeDlg", "Installed AND NOT RESUME AND NOT Preselected", 1250),
- ("ProgressDlg", None, 1280)])
-
- add_data(db, 'ActionText', text.ActionText)
- add_data(db, 'UIText', text.UIText)
- #####################################################################
- # Standard dialogs: FatalError, UserExit, ExitDialog
- fatal=PyDialog(db, "FatalError", x, y, w, h, modal, title,
- "Finish", "Finish", "Finish")
- fatal.title("[ProductName] Installer ended prematurely")
- fatal.back("< Back", "Finish", active = 0)
- fatal.cancel("Cancel", "Back", active = 0)
- fatal.text("Description1", 15, 70, 320, 80, 0x30003,
- "[ProductName] setup ended prematurely because of an error. Your system has not been modified. To install this program at a later time, please run the installation again.")
- fatal.text("Description2", 15, 155, 320, 20, 0x30003,
- "Click the Finish button to exit the Installer.")
- c=fatal.next("Finish", "Cancel", name="Finish")
- c.event("EndDialog", "Exit")
-
- user_exit=PyDialog(db, "UserExit", x, y, w, h, modal, title,
- "Finish", "Finish", "Finish")
- user_exit.title("[ProductName] Installer was interrupted")
- user_exit.back("< Back", "Finish", active = 0)
- user_exit.cancel("Cancel", "Back", active = 0)
- user_exit.text("Description1", 15, 70, 320, 80, 0x30003,
- "[ProductName] setup was interrupted. Your system has not been modified. "
- "To install this program at a later time, please run the installation again.")
- user_exit.text("Description2", 15, 155, 320, 20, 0x30003,
- "Click the Finish button to exit the Installer.")
- c = user_exit.next("Finish", "Cancel", name="Finish")
- c.event("EndDialog", "Exit")
-
- exit_dialog = PyDialog(db, "ExitDialog", x, y, w, h, modal, title,
- "Finish", "Finish", "Finish")
- exit_dialog.title("Completing the [ProductName] Installer")
- exit_dialog.back("< Back", "Finish", active = 0)
- exit_dialog.cancel("Cancel", "Back", active = 0)
- exit_dialog.text("Description", 15, 235, 320, 20, 0x30003,
- "Click the Finish button to exit the Installer.")
- c = exit_dialog.next("Finish", "Cancel", name="Finish")
- c.event("EndDialog", "Return")
-
- #####################################################################
- # Required dialog: FilesInUse, ErrorDlg
- inuse = PyDialog(db, "FilesInUse",
- x, y, w, h,
- 19, # KeepModeless|Modal|Visible
- title,
- "Retry", "Retry", "Retry", bitmap=False)
- inuse.text("Title", 15, 6, 200, 15, 0x30003,
- r"{\DlgFontBold8}Files in Use")
- inuse.text("Description", 20, 23, 280, 20, 0x30003,
- "Some files that need to be updated are currently in use.")
- inuse.text("Text", 20, 55, 330, 50, 3,
- "The following applications are using files that need to be updated by this setup. Close these applications and then click Retry to continue the installation or Cancel to exit it.")
- inuse.control("List", "ListBox", 20, 107, 330, 130, 7, "FileInUseProcess",
- None, None, None)
- c=inuse.back("Exit", "Ignore", name="Exit")
- c.event("EndDialog", "Exit")
- c=inuse.next("Ignore", "Retry", name="Ignore")
- c.event("EndDialog", "Ignore")
- c=inuse.cancel("Retry", "Exit", name="Retry")
- c.event("EndDialog","Retry")
-
- # See "Error Dialog". See "ICE20" for the required names of the controls.
- error = Dialog(db, "ErrorDlg",
- 50, 10, 330, 101,
- 65543, # Error|Minimize|Modal|Visible
- title,
- "ErrorText", None, None)
- error.text("ErrorText", 50,9,280,48,3, "")
- #error.control("ErrorIcon", "Icon", 15, 9, 24, 24, 5242881, None, "py.ico", None, None)
- error.pushbutton("N",120,72,81,21,3,"No",None).event("EndDialog","ErrorNo")
- error.pushbutton("Y",240,72,81,21,3,"Yes",None).event("EndDialog","ErrorYes")
- error.pushbutton("A",0,72,81,21,3,"Abort",None).event("EndDialog","ErrorAbort")
- error.pushbutton("C",42,72,81,21,3,"Cancel",None).event("EndDialog","ErrorCancel")
- error.pushbutton("I",81,72,81,21,3,"Ignore",None).event("EndDialog","ErrorIgnore")
- error.pushbutton("O",159,72,81,21,3,"Ok",None).event("EndDialog","ErrorOk")
- error.pushbutton("R",198,72,81,21,3,"Retry",None).event("EndDialog","ErrorRetry")
-
- #####################################################################
- # Global "Query Cancel" dialog
- cancel = Dialog(db, "CancelDlg", 50, 10, 260, 85, 3, title,
- "No", "No", "No")
- cancel.text("Text", 48, 15, 194, 30, 3,
- "Are you sure you want to cancel [ProductName] installation?")
- #cancel.control("Icon", "Icon", 15, 15, 24, 24, 5242881, None,
- # "py.ico", None, None)
- c=cancel.pushbutton("Yes", 72, 57, 56, 17, 3, "Yes", "No")
- c.event("EndDialog", "Exit")
-
- c=cancel.pushbutton("No", 132, 57, 56, 17, 3, "No", "Yes")
- c.event("EndDialog", "Return")
-
- #####################################################################
- # Global "Wait for costing" dialog
- costing = Dialog(db, "WaitForCostingDlg", 50, 10, 260, 85, modal, title,
- "Return", "Return", "Return")
- costing.text("Text", 48, 15, 194, 30, 3,
- "Please wait while the installer finishes determining your disk space requirements.")
- c = costing.pushbutton("Return", 102, 57, 56, 17, 3, "Return", None)
- c.event("EndDialog", "Exit")
-
- #####################################################################
- # Preparation dialog: no user input except cancellation
- prep = PyDialog(db, "PrepareDlg", x, y, w, h, modeless, title,
- "Cancel", "Cancel", "Cancel")
- prep.text("Description", 15, 70, 320, 40, 0x30003,
- "Please wait while the Installer prepares to guide you through the installation.")
- prep.title("Welcome to the [ProductName] Installer")
- c=prep.text("ActionText", 15, 110, 320, 20, 0x30003, "Pondering...")
- c.mapping("ActionText", "Text")
- c=prep.text("ActionData", 15, 135, 320, 30, 0x30003, None)
- c.mapping("ActionData", "Text")
- prep.back("Back", None, active=0)
- prep.next("Next", None, active=0)
- c=prep.cancel("Cancel", None)
- c.event("SpawnDialog", "CancelDlg")
-
- #####################################################################
- # Feature (Python directory) selection
- seldlg = PyDialog(db, "SelectFeaturesDlg", x, y, w, h, modal, title,
- "Next", "Next", "Cancel")
- seldlg.title("Select Python Installations")
-
- seldlg.text("Hint", 15, 30, 300, 20, 3,
- "Select the Python locations where %s should be installed."
- % self.distribution.get_fullname())
-
- seldlg.back("< Back", None, active=0)
- c = seldlg.next("Next >", "Cancel")
- order = 1
- c.event("[TARGETDIR]", "[SourceDir]", ordering=order)
- for version in self.versions + [self.other_version]:
- order += 1
- c.event("[TARGETDIR]", "[TARGETDIR%s]" % version,
- "FEATURE_SELECTED AND &Python%s=3" % version,
- ordering=order)
- c.event("SpawnWaitDialog", "WaitForCostingDlg", ordering=order + 1)
- c.event("EndDialog", "Return", ordering=order + 2)
- c = seldlg.cancel("Cancel", "Features")
- c.event("SpawnDialog", "CancelDlg")
-
- c = seldlg.control("Features", "SelectionTree", 15, 60, 300, 120, 3,
- "FEATURE", None, "PathEdit", None)
- c.event("[FEATURE_SELECTED]", "1")
- ver = self.other_version
- install_other_cond = "FEATURE_SELECTED AND &Python%s=3" % ver
- dont_install_other_cond = "FEATURE_SELECTED AND &Python%s<>3" % ver
-
- c = seldlg.text("Other", 15, 200, 300, 15, 3,
- "Provide an alternate Python location")
- c.condition("Enable", install_other_cond)
- c.condition("Show", install_other_cond)
- c.condition("Disable", dont_install_other_cond)
- c.condition("Hide", dont_install_other_cond)
-
- c = seldlg.control("PathEdit", "PathEdit", 15, 215, 300, 16, 1,
- "TARGETDIR" + ver, None, "Next", None)
- c.condition("Enable", install_other_cond)
- c.condition("Show", install_other_cond)
- c.condition("Disable", dont_install_other_cond)
- c.condition("Hide", dont_install_other_cond)
-
- #####################################################################
- # Disk cost
- cost = PyDialog(db, "DiskCostDlg", x, y, w, h, modal, title,
- "OK", "OK", "OK", bitmap=False)
- cost.text("Title", 15, 6, 200, 15, 0x30003,
- r"{\DlgFontBold8}Disk Space Requirements")
- cost.text("Description", 20, 20, 280, 20, 0x30003,
- "The disk space required for the installation of the selected features.")
- cost.text("Text", 20, 53, 330, 60, 3,
- "The highlighted volumes (if any) do not have enough disk space "
- "available for the currently selected features. You can either "
- "remove some files from the highlighted volumes, or choose to "
- "install less features onto local drive(s), or select different "
- "destination drive(s).")
- cost.control("VolumeList", "VolumeCostList", 20, 100, 330, 150, 393223,
- None, "{120}{70}{70}{70}{70}", None, None)
- cost.xbutton("OK", "Ok", None, 0.5).event("EndDialog", "Return")
-
- #####################################################################
- # WhichUsers Dialog. Only available on NT, and for privileged users.
- # This must be run before FindRelatedProducts, because that will
- # take into account whether the previous installation was per-user
- # or per-machine. We currently don't support going back to this
- # dialog after "Next" was selected; to support this, we would need to
- # find how to reset the ALLUSERS property, and how to re-run
- # FindRelatedProducts.
- # On Windows9x, the ALLUSERS property is ignored on the command line
- # and in the Property table, but installer fails according to the documentation
- # if a dialog attempts to set ALLUSERS.
- whichusers = PyDialog(db, "WhichUsersDlg", x, y, w, h, modal, title,
- "AdminInstall", "Next", "Cancel")
- whichusers.title("Select whether to install [ProductName] for all users of this computer.")
- # A radio group with two options: allusers, justme
- g = whichusers.radiogroup("AdminInstall", 15, 60, 260, 50, 3,
- "WhichUsers", "", "Next")
- g.add("ALL", 0, 5, 150, 20, "Install for all users")
- g.add("JUSTME", 0, 25, 150, 20, "Install just for me")
-
- whichusers.back("Back", None, active=0)
-
- c = whichusers.next("Next >", "Cancel")
- c.event("[ALLUSERS]", "1", 'WhichUsers="ALL"', 1)
- c.event("EndDialog", "Return", ordering = 2)
-
- c = whichusers.cancel("Cancel", "AdminInstall")
- c.event("SpawnDialog", "CancelDlg")
-
- #####################################################################
- # Installation Progress dialog (modeless)
- progress = PyDialog(db, "ProgressDlg", x, y, w, h, modeless, title,
- "Cancel", "Cancel", "Cancel", bitmap=False)
- progress.text("Title", 20, 15, 200, 15, 0x30003,
- r"{\DlgFontBold8}[Progress1] [ProductName]")
- progress.text("Text", 35, 65, 300, 30, 3,
- "Please wait while the Installer [Progress2] [ProductName]. "
- "This may take several minutes.")
- progress.text("StatusLabel", 35, 100, 35, 20, 3, "Status:")
-
- c=progress.text("ActionText", 70, 100, w-70, 20, 3, "Pondering...")
- c.mapping("ActionText", "Text")
-
- #c=progress.text("ActionData", 35, 140, 300, 20, 3, None)
- #c.mapping("ActionData", "Text")
-
- c=progress.control("ProgressBar", "ProgressBar", 35, 120, 300, 10, 65537,
- None, "Progress done", None, None)
- c.mapping("SetProgress", "Progress")
-
- progress.back("< Back", "Next", active=False)
- progress.next("Next >", "Cancel", active=False)
- progress.cancel("Cancel", "Back").event("SpawnDialog", "CancelDlg")
-
- ###################################################################
- # Maintenance type: repair/uninstall
- maint = PyDialog(db, "MaintenanceTypeDlg", x, y, w, h, modal, title,
- "Next", "Next", "Cancel")
- maint.title("Welcome to the [ProductName] Setup Wizard")
- maint.text("BodyText", 15, 63, 330, 42, 3,
- "Select whether you want to repair or remove [ProductName].")
- g=maint.radiogroup("RepairRadioGroup", 15, 108, 330, 60, 3,
- "MaintenanceForm_Action", "", "Next")
- #g.add("Change", 0, 0, 200, 17, "&Change [ProductName]")
- g.add("Repair", 0, 18, 200, 17, "&Repair [ProductName]")
- g.add("Remove", 0, 36, 200, 17, "Re&move [ProductName]")
-
- maint.back("< Back", None, active=False)
- c=maint.next("Finish", "Cancel")
- # Change installation: Change progress dialog to "Change", then ask
- # for feature selection
- #c.event("[Progress1]", "Change", 'MaintenanceForm_Action="Change"', 1)
- #c.event("[Progress2]", "changes", 'MaintenanceForm_Action="Change"', 2)
-
- # Reinstall: Change progress dialog to "Repair", then invoke reinstall
- # Also set list of reinstalled features to "ALL"
- c.event("[REINSTALL]", "ALL", 'MaintenanceForm_Action="Repair"', 5)
- c.event("[Progress1]", "Repairing", 'MaintenanceForm_Action="Repair"', 6)
- c.event("[Progress2]", "repairs", 'MaintenanceForm_Action="Repair"', 7)
- c.event("Reinstall", "ALL", 'MaintenanceForm_Action="Repair"', 8)
-
- # Uninstall: Change progress to "Remove", then invoke uninstall
- # Also set list of removed features to "ALL"
- c.event("[REMOVE]", "ALL", 'MaintenanceForm_Action="Remove"', 11)
- c.event("[Progress1]", "Removing", 'MaintenanceForm_Action="Remove"', 12)
- c.event("[Progress2]", "removes", 'MaintenanceForm_Action="Remove"', 13)
- c.event("Remove", "ALL", 'MaintenanceForm_Action="Remove"', 14)
-
- # Close dialog when maintenance action scheduled
- c.event("EndDialog", "Return", 'MaintenanceForm_Action<>"Change"', 20)
- #c.event("NewDialog", "SelectFeaturesDlg", 'MaintenanceForm_Action="Change"', 21)
-
- maint.cancel("Cancel", "RepairRadioGroup").event("SpawnDialog", "CancelDlg")
-
- def get_installer_filename(self, fullname):
- # Factored out to allow overriding in subclasses
- if self.target_version:
- base_name = "%s.%s-py%s.msi" % (fullname, self.plat_name,
- self.target_version)
- else:
- base_name = "%s.%s.msi" % (fullname, self.plat_name)
- installer_name = os.path.join(self.dist_dir, base_name)
- return installer_name
diff --git a/common/py3-stdlib/distutils/command/build_ext.py b/common/py3-stdlib/distutils/command/build_ext.py
index 1a9bd12..f287b34 100644
--- a/common/py3-stdlib/distutils/command/build_ext.py
+++ b/common/py3-stdlib/distutils/command/build_ext.py
@@ -692,7 +692,7 @@
suffix = '_' + ext.name.split('.')[-1]
try:
# Unicode module name support as defined in PEP-489
- # https://www.python.org/dev/peps/pep-0489/#export-hook-name
+ # https://peps.python.org/pep-0489/#export-hook-name
suffix.encode('ascii')
except UnicodeEncodeError:
suffix = 'U' + suffix.encode('punycode').replace(b'-', b'_').decode('ascii')
diff --git a/common/py3-stdlib/distutils/config.py b/common/py3-stdlib/distutils/config.py
index 2171abd..a201c86 100644
--- a/common/py3-stdlib/distutils/config.py
+++ b/common/py3-stdlib/distutils/config.py
@@ -5,6 +5,7 @@
"""
import os
from configparser import RawConfigParser
+import warnings
from distutils.cmd import Command
@@ -111,7 +112,9 @@
def _read_pypi_response(self, response):
"""Read and decode a PyPI HTTP response."""
- import cgi
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+ import cgi
content_type = response.getheader('content-type', 'text/plain')
encoding = cgi.parse_header(content_type)[1].get('charset', 'ascii')
return response.read().decode(encoding)
diff --git a/common/py3-stdlib/distutils/msvccompiler.py b/common/py3-stdlib/distutils/msvccompiler.py
index d5857cb..f0d04fd 100644
--- a/common/py3-stdlib/distutils/msvccompiler.py
+++ b/common/py3-stdlib/distutils/msvccompiler.py
@@ -44,7 +44,6 @@
"necessary compiler setting\n"
"Make sure that Python modules winreg, "
"win32api or win32con are installed.")
- pass
if _can_read_reg:
HKEYS = (hkey_mod.HKEY_USERS,
diff --git a/common/py3-stdlib/distutils/sysconfig.py b/common/py3-stdlib/distutils/sysconfig.py
index 3414a76..03b8558 100644
--- a/common/py3-stdlib/distutils/sysconfig.py
+++ b/common/py3-stdlib/distutils/sysconfig.py
@@ -30,8 +30,6 @@
parse_config_h as sysconfig_parse_config_h,
_init_non_posix,
- _is_python_source_dir,
- _sys_home,
_variable_rx,
_findvar1_rx,
@@ -52,9 +50,6 @@
# which might not be true in the time of import.
_config_vars = get_config_vars()
-if os.name == "nt":
- from sysconfig import _fix_pcbuild
-
warnings.warn(
'The distutils.sysconfig module is deprecated, use sysconfig instead',
DeprecationWarning,
@@ -287,7 +282,7 @@
# must use "srcdir" from the makefile to find the "Include"
# directory.
if plat_specific:
- return _sys_home or project_base
+ return project_base
else:
incdir = os.path.join(get_config_var('srcdir'), 'Include')
return os.path.normpath(incdir)
diff --git a/common/py3-stdlib/distutils/tests/test_archive_util.py b/common/py3-stdlib/distutils/tests/test_archive_util.py
index edcec25..8aec840 100644
--- a/common/py3-stdlib/distutils/tests/test_archive_util.py
+++ b/common/py3-stdlib/distutils/tests/test_archive_util.py
@@ -390,7 +390,7 @@
archive.close()
def test_suite():
- return unittest.makeSuite(ArchiveUtilTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(ArchiveUtilTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_bdist.py b/common/py3-stdlib/distutils/tests/test_bdist.py
index 55fa393..241fc9a 100644
--- a/common/py3-stdlib/distutils/tests/test_bdist.py
+++ b/common/py3-stdlib/distutils/tests/test_bdist.py
@@ -18,13 +18,12 @@
# we can set the format
dist = self.create_dist()[1]
cmd = bdist(dist)
- cmd.formats = ['msi']
+ cmd.formats = ['tar']
cmd.ensure_finalized()
- self.assertEqual(cmd.formats, ['msi'])
+ self.assertEqual(cmd.formats, ['tar'])
# what formats does bdist offer?
- formats = ['bztar', 'gztar', 'msi', 'rpm', 'tar',
- 'xztar', 'zip', 'ztar']
+ formats = ['bztar', 'gztar', 'rpm', 'tar', 'xztar', 'zip', 'ztar']
found = sorted(cmd.format_command)
self.assertEqual(found, formats)
@@ -36,11 +35,7 @@
cmd.ensure_finalized()
dist.command_obj['bdist'] = cmd
- names = ['bdist_dumb'] # bdist_rpm does not support --skip-build
- if os.name == 'nt':
- names.append('bdist_msi')
-
- for name in names:
+ for name in ['bdist_dumb']: # bdist_rpm does not support --skip-build
subcmd = cmd.get_finalized_command(name)
if getattr(subcmd, '_unsupported', False):
# command is not supported on this build
@@ -50,7 +45,8 @@
def test_suite():
- return unittest.makeSuite(BuildTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(BuildTestCase)
+
if __name__ == '__main__':
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_bdist_dumb.py b/common/py3-stdlib/distutils/tests/test_bdist_dumb.py
index 01a233b..bb860c8 100644
--- a/common/py3-stdlib/distutils/tests/test_bdist_dumb.py
+++ b/common/py3-stdlib/distutils/tests/test_bdist_dumb.py
@@ -91,7 +91,7 @@
self.assertEqual(contents, sorted(wanted))
def test_suite():
- return unittest.makeSuite(BuildDumbTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(BuildDumbTestCase)
if __name__ == '__main__':
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_bdist_msi.py b/common/py3-stdlib/distutils/tests/test_bdist_msi.py
deleted file mode 100644
index a61266a..0000000
--- a/common/py3-stdlib/distutils/tests/test_bdist_msi.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""Tests for distutils.command.bdist_msi."""
-import sys
-import unittest
-from test.support import run_unittest
-from test.support.warnings_helper import check_warnings
-from distutils.tests import support
-
-
-@unittest.skipUnless(sys.platform == 'win32', 'these tests require Windows')
-class BDistMSITestCase(support.TempdirManager,
- support.LoggingSilencer,
- unittest.TestCase):
-
- def test_minimal(self):
- # minimal test XXX need more tests
- from distutils.command.bdist_msi import bdist_msi
- project_dir, dist = self.create_dist()
- with check_warnings(("", DeprecationWarning)):
- cmd = bdist_msi(dist)
- cmd.ensure_finalized()
-
-
-def test_suite():
- return unittest.makeSuite(BDistMSITestCase)
-
-if __name__ == '__main__':
- run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_bdist_rpm.py b/common/py3-stdlib/distutils/tests/test_bdist_rpm.py
index ba4382f..7eefa7b 100644
--- a/common/py3-stdlib/distutils/tests/test_bdist_rpm.py
+++ b/common/py3-stdlib/distutils/tests/test_bdist_rpm.py
@@ -49,6 +49,9 @@
'the rpm command is not found')
@unittest.skipIf(find_executable('rpmbuild') is None,
'the rpmbuild command is not found')
+ # import foo fails with safe path
+ @unittest.skipIf(sys.flags.safe_path,
+ 'PYTHONSAFEPATH changes default sys.path')
def test_quiet(self):
# let's create a package
tmp_dir = self.mkdtemp()
@@ -93,6 +96,9 @@
'the rpm command is not found')
@unittest.skipIf(find_executable('rpmbuild') is None,
'the rpmbuild command is not found')
+ # import foo fails with safe path
+ @unittest.skipIf(sys.flags.safe_path,
+ 'PYTHONSAFEPATH changes default sys.path')
def test_no_optimize_flag(self):
# let's create a package that breaks bdist_rpm
tmp_dir = self.mkdtemp()
@@ -129,7 +135,7 @@
os.remove(os.path.join(pkg_dir, 'dist', 'foo-0.1-1.noarch.rpm'))
def test_suite():
- return unittest.makeSuite(BuildRpmTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(BuildRpmTestCase)
if __name__ == '__main__':
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_build.py b/common/py3-stdlib/distutils/tests/test_build.py
index b020a5b..71b5e16 100644
--- a/common/py3-stdlib/distutils/tests/test_build.py
+++ b/common/py3-stdlib/distutils/tests/test_build.py
@@ -12,6 +12,7 @@
support.LoggingSilencer,
unittest.TestCase):
+ @unittest.skipUnless(sys.executable, "test requires sys.executable")
def test_finalize_options(self):
pkg_dir, dist = self.create_dist()
cmd = build(dist)
@@ -50,7 +51,7 @@
self.assertEqual(cmd.executable, os.path.normpath(sys.executable))
def test_suite():
- return unittest.makeSuite(BuildTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(BuildTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_build_clib.py b/common/py3-stdlib/distutils/tests/test_build_clib.py
index 19e012a..95f9282 100644
--- a/common/py3-stdlib/distutils/tests/test_build_clib.py
+++ b/common/py3-stdlib/distutils/tests/test_build_clib.py
@@ -4,7 +4,9 @@
import sys
import sysconfig
-from test.support import run_unittest, missing_compiler_executable
+from test.support import (
+ run_unittest, missing_compiler_executable, requires_subprocess
+)
from distutils.command.build_clib import build_clib
from distutils.errors import DistutilsSetupError
@@ -112,6 +114,7 @@
self.assertRaises(DistutilsSetupError, cmd.finalize_options)
@unittest.skipIf(sys.platform == 'win32', "can't test on Windows")
+ @requires_subprocess()
def test_run(self):
pkg_dir, dist = self.create_dist()
cmd = build_clib(dist)
@@ -138,7 +141,7 @@
self.assertIn('libfoo.a', os.listdir(build_temp))
def test_suite():
- return unittest.makeSuite(BuildCLibTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(BuildCLibTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_build_ext.py b/common/py3-stdlib/distutils/tests/test_build_ext.py
index 8e7364d..4ebeafe 100644
--- a/common/py3-stdlib/distutils/tests/test_build_ext.py
+++ b/common/py3-stdlib/distutils/tests/test_build_ext.py
@@ -17,6 +17,7 @@
from test import support
from test.support import os_helper
from test.support.script_helper import assert_python_ok
+from test.support import threading_helper
# http://bugs.python.org/issue4373
# Don't load the xx module more than once.
@@ -40,9 +41,7 @@
# bpo-30132: On Windows, a .pdb file may be created in the current
# working directory. Create a temporary working directory to cleanup
# everything at the end of the test.
- change_cwd = os_helper.change_cwd(self.tmp_dir)
- change_cwd.__enter__()
- self.addCleanup(change_cwd.__exit__, None, None, None)
+ self.enterContext(os_helper.change_cwd(self.tmp_dir))
def tearDown(self):
import site
@@ -56,6 +55,7 @@
def build_ext(self, *args, **kwargs):
return build_ext(*args, **kwargs)
+ @support.requires_subprocess()
def test_build_ext(self):
cmd = support.missing_compiler_executable()
if cmd is not None:
@@ -164,6 +164,7 @@
self.assertIn(lib, cmd.rpath)
self.assertIn(incl, cmd.include_dirs)
+ @threading_helper.requires_working_threading()
def test_optional_extension(self):
# this extension will fail, but let's ignore this failure
@@ -332,6 +333,7 @@
cmd.run()
self.assertEqual(cmd.compiler, 'unix')
+ @support.requires_subprocess()
def test_get_outputs(self):
cmd = support.missing_compiler_executable()
if cmd is not None:
@@ -545,8 +547,8 @@
def test_suite():
suite = unittest.TestSuite()
- suite.addTest(unittest.makeSuite(BuildExtTestCase))
- suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase))
+ suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BuildExtTestCase))
+ suite.addTest(unittest.TestLoader().loadTestsFromTestCase(ParallelBuildExtTestCase))
return suite
if __name__ == '__main__':
diff --git a/common/py3-stdlib/distutils/tests/test_build_py.py b/common/py3-stdlib/distutils/tests/test_build_py.py
index 0712e92..44a06cc 100644
--- a/common/py3-stdlib/distutils/tests/test_build_py.py
+++ b/common/py3-stdlib/distutils/tests/test_build_py.py
@@ -9,7 +9,7 @@
from distutils.errors import DistutilsFileError
from distutils.tests import support
-from test.support import run_unittest
+from test.support import run_unittest, requires_subprocess
class BuildPyTestCase(support.TempdirManager,
@@ -89,6 +89,7 @@
self.fail("failed package_data test when package_dir is ''")
@unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
+ @requires_subprocess()
def test_byte_compile(self):
project_dir, dist = self.create_dist(py_modules=['boiledeggs'])
os.chdir(project_dir)
@@ -106,6 +107,7 @@
['boiledeggs.%s.pyc' % sys.implementation.cache_tag])
@unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
+ @requires_subprocess()
def test_byte_compile_optimized(self):
project_dir, dist = self.create_dist(py_modules=['boiledeggs'])
os.chdir(project_dir)
@@ -173,7 +175,7 @@
def test_suite():
- return unittest.makeSuite(BuildPyTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(BuildPyTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_build_scripts.py b/common/py3-stdlib/distutils/tests/test_build_scripts.py
index 954fc76..f299e51 100644
--- a/common/py3-stdlib/distutils/tests/test_build_scripts.py
+++ b/common/py3-stdlib/distutils/tests/test_build_scripts.py
@@ -106,7 +106,7 @@
self.assertIn(name, built)
def test_suite():
- return unittest.makeSuite(BuildScriptsTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(BuildScriptsTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_check.py b/common/py3-stdlib/distutils/tests/test_check.py
index e534aca..91bcdce 100644
--- a/common/py3-stdlib/distutils/tests/test_check.py
+++ b/common/py3-stdlib/distutils/tests/test_check.py
@@ -157,7 +157,7 @@
'restructuredtext': 1})
def test_suite():
- return unittest.makeSuite(CheckTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(CheckTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_clean.py b/common/py3-stdlib/distutils/tests/test_clean.py
index c605afd..9236749 100644
--- a/common/py3-stdlib/distutils/tests/test_clean.py
+++ b/common/py3-stdlib/distutils/tests/test_clean.py
@@ -43,7 +43,7 @@
cmd.run()
def test_suite():
- return unittest.makeSuite(cleanTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(cleanTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_cmd.py b/common/py3-stdlib/distutils/tests/test_cmd.py
index cf5197c..2319214 100644
--- a/common/py3-stdlib/distutils/tests/test_cmd.py
+++ b/common/py3-stdlib/distutils/tests/test_cmd.py
@@ -120,7 +120,7 @@
debug.DEBUG = False
def test_suite():
- return unittest.makeSuite(CommandTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(CommandTestCase)
if __name__ == '__main__':
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_config.py b/common/py3-stdlib/distutils/tests/test_config.py
index 344084a..8ab70ef 100644
--- a/common/py3-stdlib/distutils/tests/test_config.py
+++ b/common/py3-stdlib/distutils/tests/test_config.py
@@ -135,7 +135,7 @@
def test_suite():
- return unittest.makeSuite(PyPIRCCommandTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(PyPIRCCommandTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_config_cmd.py b/common/py3-stdlib/distutils/tests/test_config_cmd.py
index 0127ba7..c79db68 100644
--- a/common/py3-stdlib/distutils/tests/test_config_cmd.py
+++ b/common/py3-stdlib/distutils/tests/test_config_cmd.py
@@ -3,7 +3,9 @@
import os
import sys
import sysconfig
-from test.support import run_unittest, missing_compiler_executable
+from test.support import (
+ run_unittest, missing_compiler_executable, requires_subprocess
+)
from distutils.command.config import dump_file, config
from distutils.tests import support
@@ -42,6 +44,7 @@
self.assertEqual(len(self._logs), numlines+1)
@unittest.skipIf(sys.platform == 'win32', "can't test on Windows")
+ @requires_subprocess()
def test_search_cpp(self):
cmd = missing_compiler_executable(['preprocessor'])
if cmd is not None:
@@ -94,7 +97,7 @@
self.assertFalse(os.path.exists(f))
def test_suite():
- return unittest.makeSuite(ConfigTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(ConfigTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_core.py b/common/py3-stdlib/distutils/tests/test_core.py
index 4e6694a..700a22d 100644
--- a/common/py3-stdlib/distutils/tests/test_core.py
+++ b/common/py3-stdlib/distutils/tests/test_core.py
@@ -134,7 +134,7 @@
self.assertEqual(stdout.readlines()[0], wanted)
def test_suite():
- return unittest.makeSuite(CoreTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(CoreTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_cygwinccompiler.py b/common/py3-stdlib/distutils/tests/test_cygwinccompiler.py
index 9dc869d..0912ffd 100644
--- a/common/py3-stdlib/distutils/tests/test_cygwinccompiler.py
+++ b/common/py3-stdlib/distutils/tests/test_cygwinccompiler.py
@@ -148,7 +148,7 @@
self.assertRaises(ValueError, get_msvcr)
def test_suite():
- return unittest.makeSuite(CygwinCCompilerTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(CygwinCCompilerTestCase)
if __name__ == '__main__':
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_dep_util.py b/common/py3-stdlib/distutils/tests/test_dep_util.py
index c6fae39..0d52740 100644
--- a/common/py3-stdlib/distutils/tests/test_dep_util.py
+++ b/common/py3-stdlib/distutils/tests/test_dep_util.py
@@ -74,7 +74,7 @@
def test_suite():
- return unittest.makeSuite(DepUtilTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(DepUtilTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_dir_util.py b/common/py3-stdlib/distutils/tests/test_dir_util.py
index d436cf8..ebd89f3 100644
--- a/common/py3-stdlib/distutils/tests/test_dir_util.py
+++ b/common/py3-stdlib/distutils/tests/test_dir_util.py
@@ -11,7 +11,7 @@
from distutils import log
from distutils.tests import support
-from test.support import run_unittest
+from test.support import run_unittest, is_emscripten, is_wasi
class DirUtilTestCase(support.TempdirManager, unittest.TestCase):
@@ -55,6 +55,10 @@
@unittest.skipIf(sys.platform.startswith('win'),
"This test is only appropriate for POSIX-like systems.")
+ @unittest.skipIf(
+ is_emscripten or is_wasi,
+ "Emscripten's/WASI's umask is a stub."
+ )
def test_mkpath_with_custom_mode(self):
# Get and set the current umask value for testing mode bits.
umask = os.umask(0o002)
@@ -133,7 +137,7 @@
def test_suite():
- return unittest.makeSuite(DirUtilTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(DirUtilTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_dist.py b/common/py3-stdlib/distutils/tests/test_dist.py
index f8a9e86..2ef70d9 100644
--- a/common/py3-stdlib/distutils/tests/test_dist.py
+++ b/common/py3-stdlib/distutils/tests/test_dist.py
@@ -521,8 +521,8 @@
def test_suite():
suite = unittest.TestSuite()
- suite.addTest(unittest.makeSuite(DistributionTestCase))
- suite.addTest(unittest.makeSuite(MetadataTestCase))
+ suite.addTest(unittest.TestLoader().loadTestsFromTestCase(DistributionTestCase))
+ suite.addTest(unittest.TestLoader().loadTestsFromTestCase(MetadataTestCase))
return suite
if __name__ == "__main__":
diff --git a/common/py3-stdlib/distutils/tests/test_extension.py b/common/py3-stdlib/distutils/tests/test_extension.py
index 81fad02..2b08930 100644
--- a/common/py3-stdlib/distutils/tests/test_extension.py
+++ b/common/py3-stdlib/distutils/tests/test_extension.py
@@ -64,7 +64,7 @@
"Unknown Extension options: 'chic'")
def test_suite():
- return unittest.makeSuite(ExtensionTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(ExtensionTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_file_util.py b/common/py3-stdlib/distutils/tests/test_file_util.py
index c7783b8..551151b 100644
--- a/common/py3-stdlib/distutils/tests/test_file_util.py
+++ b/common/py3-stdlib/distutils/tests/test_file_util.py
@@ -79,6 +79,7 @@
fobj.write('spam eggs')
move_file(self.source, self.target, verbose=0)
+ @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link')
def test_copy_file_hard_link(self):
with open(self.source, 'w') as f:
f.write('some content')
@@ -99,6 +100,7 @@
with open(self.source, 'r') as f:
self.assertEqual(f.read(), 'some content')
+ @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link')
def test_copy_file_hard_link_failure(self):
# If hard linking fails, copy_file() falls back on copying file
# (some special filesystems don't support hard linking even under
@@ -118,7 +120,7 @@
def test_suite():
- return unittest.makeSuite(FileUtilTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(FileUtilTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_filelist.py b/common/py3-stdlib/distutils/tests/test_filelist.py
index cee97d4..98c97e4 100644
--- a/common/py3-stdlib/distutils/tests/test_filelist.py
+++ b/common/py3-stdlib/distutils/tests/test_filelist.py
@@ -331,8 +331,8 @@
def test_suite():
return unittest.TestSuite([
- unittest.makeSuite(FileListTestCase),
- unittest.makeSuite(FindAllTestCase),
+ unittest.TestLoader().loadTestsFromTestCase(FileListTestCase),
+ unittest.TestLoader().loadTestsFromTestCase(FindAllTestCase),
])
diff --git a/common/py3-stdlib/distutils/tests/test_install.py b/common/py3-stdlib/distutils/tests/test_install.py
index 0632024..c38f98b 100644
--- a/common/py3-stdlib/distutils/tests/test_install.py
+++ b/common/py3-stdlib/distutils/tests/test_install.py
@@ -5,7 +5,7 @@
import unittest
import site
-from test.support import captured_stdout, run_unittest
+from test.support import captured_stdout, run_unittest, requires_subprocess
from distutils import sysconfig
from distutils.command.install import install, HAS_USER_SITE
@@ -208,6 +208,7 @@
'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]]
self.assertEqual(found, expected)
+ @requires_subprocess()
def test_record_extensions(self):
cmd = test_support.missing_compiler_executable()
if cmd is not None:
@@ -254,7 +255,7 @@
def test_suite():
- return unittest.makeSuite(InstallTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(InstallTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_install_data.py b/common/py3-stdlib/distutils/tests/test_install_data.py
index 32ab296..6191d2f 100644
--- a/common/py3-stdlib/distutils/tests/test_install_data.py
+++ b/common/py3-stdlib/distutils/tests/test_install_data.py
@@ -69,7 +69,7 @@
self.assertTrue(os.path.exists(os.path.join(inst, rone)))
def test_suite():
- return unittest.makeSuite(InstallDataTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(InstallDataTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_install_headers.py b/common/py3-stdlib/distutils/tests/test_install_headers.py
index 2217b32..1aa4d09 100644
--- a/common/py3-stdlib/distutils/tests/test_install_headers.py
+++ b/common/py3-stdlib/distutils/tests/test_install_headers.py
@@ -33,7 +33,7 @@
self.assertEqual(len(cmd.get_outputs()), 2)
def test_suite():
- return unittest.makeSuite(InstallHeadersTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(InstallHeadersTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_install_lib.py b/common/py3-stdlib/distutils/tests/test_install_lib.py
index fda6315..f840d1a 100644
--- a/common/py3-stdlib/distutils/tests/test_install_lib.py
+++ b/common/py3-stdlib/distutils/tests/test_install_lib.py
@@ -8,7 +8,7 @@
from distutils.extension import Extension
from distutils.tests import support
from distutils.errors import DistutilsOptionError
-from test.support import run_unittest
+from test.support import run_unittest, requires_subprocess
class InstallLibTestCase(support.TempdirManager,
@@ -35,6 +35,7 @@
self.assertEqual(cmd.optimize, 2)
@unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
+ @requires_subprocess()
def test_byte_compile(self):
project_dir, dist = self.create_dist()
os.chdir(project_dir)
@@ -90,6 +91,7 @@
inputs = cmd.get_inputs()
self.assertEqual(len(inputs), 2, inputs)
+ @requires_subprocess()
def test_dont_write_bytecode(self):
# makes sure byte_compile is not used
dist = self.create_dist()[1]
@@ -109,7 +111,7 @@
def test_suite():
- return unittest.makeSuite(InstallLibTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(InstallLibTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_install_scripts.py b/common/py3-stdlib/distutils/tests/test_install_scripts.py
index 1f7b103..648db3b 100644
--- a/common/py3-stdlib/distutils/tests/test_install_scripts.py
+++ b/common/py3-stdlib/distutils/tests/test_install_scripts.py
@@ -76,7 +76,7 @@
def test_suite():
- return unittest.makeSuite(InstallScriptsTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(InstallScriptsTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_log.py b/common/py3-stdlib/distutils/tests/test_log.py
index 75cf900..ec2ae02 100644
--- a/common/py3-stdlib/distutils/tests/test_log.py
+++ b/common/py3-stdlib/distutils/tests/test_log.py
@@ -40,7 +40,7 @@
'Fαtal\t\\xc8rr\\u014dr')
def test_suite():
- return unittest.makeSuite(TestLog)
+ return unittest.TestLoader().loadTestsFromTestCase(TestLog)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_msvc9compiler.py b/common/py3-stdlib/distutils/tests/test_msvc9compiler.py
index 77a07ef..6235405 100644
--- a/common/py3-stdlib/distutils/tests/test_msvc9compiler.py
+++ b/common/py3-stdlib/distutils/tests/test_msvc9compiler.py
@@ -178,7 +178,7 @@
def test_suite():
- return unittest.makeSuite(msvc9compilerTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(msvc9compilerTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_msvccompiler.py b/common/py3-stdlib/distutils/tests/test_msvccompiler.py
index b518d6a..dd67c3e 100644
--- a/common/py3-stdlib/distutils/tests/test_msvccompiler.py
+++ b/common/py3-stdlib/distutils/tests/test_msvccompiler.py
@@ -75,7 +75,7 @@
raise unittest.SkipTest("VS 2015 is not installed")
def test_suite():
- return unittest.makeSuite(msvccompilerTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(msvccompilerTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_register.py b/common/py3-stdlib/distutils/tests/test_register.py
index bba4863..7805c6d 100644
--- a/common/py3-stdlib/distutils/tests/test_register.py
+++ b/common/py3-stdlib/distutils/tests/test_register.py
@@ -318,7 +318,7 @@
def test_suite():
- return unittest.makeSuite(RegisterTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(RegisterTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_sdist.py b/common/py3-stdlib/distutils/tests/test_sdist.py
index 752e9db..46b3a13 100644
--- a/common/py3-stdlib/distutils/tests/test_sdist.py
+++ b/common/py3-stdlib/distutils/tests/test_sdist.py
@@ -487,7 +487,7 @@
archive.close()
def test_suite():
- return unittest.makeSuite(SDistTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(SDistTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_spawn.py b/common/py3-stdlib/distutils/tests/test_spawn.py
index 4ec767b..a0a1145 100644
--- a/common/py3-stdlib/distutils/tests/test_spawn.py
+++ b/common/py3-stdlib/distutils/tests/test_spawn.py
@@ -3,7 +3,7 @@
import stat
import sys
import unittest.mock
-from test.support import run_unittest, unix_shell
+from test.support import run_unittest, unix_shell, requires_subprocess
from test.support import os_helper
from distutils.spawn import find_executable
@@ -11,6 +11,8 @@
from distutils.errors import DistutilsExecError
from distutils.tests import support
+
+@requires_subprocess()
class SpawnTestCase(support.TempdirManager,
support.LoggingSilencer,
unittest.TestCase):
@@ -131,7 +133,7 @@
def test_suite():
- return unittest.makeSuite(SpawnTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(SpawnTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_sysconfig.py b/common/py3-stdlib/distutils/tests/test_sysconfig.py
index 59676b0..6833d22 100644
--- a/common/py3-stdlib/distutils/tests/test_sysconfig.py
+++ b/common/py3-stdlib/distutils/tests/test_sysconfig.py
@@ -10,7 +10,7 @@
from distutils import sysconfig
from distutils.ccompiler import get_default_compiler
from distutils.tests import support
-from test.support import run_unittest, swap_item
+from test.support import run_unittest, swap_item, requires_subprocess, is_wasi
from test.support.os_helper import TESTFN
from test.support.warnings_helper import check_warnings
@@ -32,6 +32,7 @@
elif os.path.isdir(TESTFN):
shutil.rmtree(TESTFN)
+ @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
def test_get_config_h_filename(self):
config_h = sysconfig.get_config_h_filename()
self.assertTrue(os.path.isfile(config_h), config_h)
@@ -48,6 +49,7 @@
self.assertIsInstance(cvars, dict)
self.assertTrue(cvars)
+ @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
def test_srcdir(self):
# See Issues #15322, #15364.
srcdir = sysconfig.get_config_var('srcdir')
@@ -60,7 +62,11 @@
# should be a full source checkout.
Python_h = os.path.join(srcdir, 'Include', 'Python.h')
self.assertTrue(os.path.exists(Python_h), Python_h)
- self.assertTrue(sysconfig._is_python_source_dir(srcdir))
+ # <srcdir>/PC/pyconfig.h always exists even if unused on POSIX.
+ pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h')
+ self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h)
+ pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in')
+ self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in)
elif os.name == 'posix':
self.assertEqual(
os.path.dirname(sysconfig.get_makefile_filename()),
@@ -227,26 +233,7 @@
self.assertEqual(global_sysconfig.get_config_var('CC'),
sysconfig.get_config_var('CC'))
- @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None,
- 'EXT_SUFFIX required for this test')
- def test_SO_deprecation(self):
- self.assertWarns(DeprecationWarning,
- sysconfig.get_config_var, 'SO')
-
- @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None,
- 'EXT_SUFFIX required for this test')
- def test_SO_value(self):
- with check_warnings(('', DeprecationWarning)):
- self.assertEqual(sysconfig.get_config_var('SO'),
- sysconfig.get_config_var('EXT_SUFFIX'))
-
- @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None,
- 'EXT_SUFFIX required for this test')
- def test_SO_in_vars(self):
- vars = sysconfig.get_config_vars()
- self.assertIsNotNone(vars['SO'])
- self.assertEqual(vars['SO'], vars['EXT_SUFFIX'])
-
+ @requires_subprocess()
def test_customize_compiler_before_get_config_vars(self):
# Issue #21923: test that a Distribution compiler
# instance can be called without an explicit call to
@@ -269,7 +256,7 @@
def test_suite():
suite = unittest.TestSuite()
- suite.addTest(unittest.makeSuite(SysconfigTestCase))
+ suite.addTest(unittest.TestLoader().loadTestsFromTestCase(SysconfigTestCase))
return suite
diff --git a/common/py3-stdlib/distutils/tests/test_text_file.py b/common/py3-stdlib/distutils/tests/test_text_file.py
index 7e76240..ebac3d5 100644
--- a/common/py3-stdlib/distutils/tests/test_text_file.py
+++ b/common/py3-stdlib/distutils/tests/test_text_file.py
@@ -101,7 +101,7 @@
in_file.close()
def test_suite():
- return unittest.makeSuite(TextFileTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(TextFileTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_unixccompiler.py b/common/py3-stdlib/distutils/tests/test_unixccompiler.py
index 24725ea..a3484d4 100644
--- a/common/py3-stdlib/distutils/tests/test_unixccompiler.py
+++ b/common/py3-stdlib/distutils/tests/test_unixccompiler.py
@@ -139,7 +139,7 @@
def test_suite():
- return unittest.makeSuite(UnixCCompilerTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(UnixCCompilerTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_upload.py b/common/py3-stdlib/distutils/tests/test_upload.py
index 74f0bc0..d679741 100644
--- a/common/py3-stdlib/distutils/tests/test_upload.py
+++ b/common/py3-stdlib/distutils/tests/test_upload.py
@@ -217,7 +217,7 @@
def test_suite():
- return unittest.makeSuite(uploadTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(uploadTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_util.py b/common/py3-stdlib/distutils/tests/test_util.py
index d4a01c6..f9c223f 100644
--- a/common/py3-stdlib/distutils/tests/test_util.py
+++ b/common/py3-stdlib/distutils/tests/test_util.py
@@ -248,7 +248,10 @@
util._environ_checked = 0
os.environ.pop('HOME', None)
- import pwd
+ try:
+ import pwd
+ except ImportError:
+ raise unittest.SkipTest("Test requires pwd module.")
# only set pw_dir field, other fields are not used
result = pwd.struct_passwd((None, None, None, None, None,
@@ -304,7 +307,7 @@
def test_suite():
- return unittest.makeSuite(UtilTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(UtilTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/distutils/tests/test_version.py b/common/py3-stdlib/distutils/tests/test_version.py
index 8671cd2..1563e02 100644
--- a/common/py3-stdlib/distutils/tests/test_version.py
+++ b/common/py3-stdlib/distutils/tests/test_version.py
@@ -81,7 +81,7 @@
(v1, v2, res))
def test_suite():
- return unittest.makeSuite(VersionTestCase)
+ return unittest.TestLoader().loadTestsFromTestCase(VersionTestCase)
if __name__ == "__main__":
run_unittest(test_suite())
diff --git a/common/py3-stdlib/doctest.py b/common/py3-stdlib/doctest.py
index 37b31cf..dafad50 100644
--- a/common/py3-stdlib/doctest.py
+++ b/common/py3-stdlib/doctest.py
@@ -956,7 +956,8 @@
return module is inspect.getmodule(object)
elif inspect.isfunction(object):
return module.__dict__ is object.__globals__
- elif inspect.ismethoddescriptor(object):
+ elif (inspect.ismethoddescriptor(object) or
+ inspect.ismethodwrapper(object)):
if hasattr(object, '__objclass__'):
obj_mod = object.__objclass__.__module__
elif hasattr(object, '__module__'):
@@ -1117,7 +1118,7 @@
if inspect.istraceback(obj): obj = obj.tb_frame
if inspect.isframe(obj): obj = obj.f_code
if inspect.iscode(obj):
- lineno = getattr(obj, 'co_firstlineno', None)-1
+ lineno = obj.co_firstlineno - 1
# Find the line number where the docstring starts. Assume
# that it's the first line that begins with a quote mark.
diff --git a/common/py3-stdlib/email/charset.py b/common/py3-stdlib/email/charset.py
index 791b658..d3d759a 100644
--- a/common/py3-stdlib/email/charset.py
+++ b/common/py3-stdlib/email/charset.py
@@ -112,8 +112,8 @@
charset is the input character set, and must be the canonical name of a
character set.
- Optional header_enc and body_enc is either charset.QP for
- quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for
+ Optional header_enc and body_enc is either Charset.QP for
+ quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
is only valid for header_enc. It describes how message headers and
message bodies in the input charset are to be encoded. Default is no
@@ -185,13 +185,13 @@
header_encoding: If the character set must be encoded before it can be
used in an email header, this attribute will be set to
- charset.QP (for quoted-printable), charset.BASE64 (for
- base64 encoding), or charset.SHORTEST for the shortest of
+ Charset.QP (for quoted-printable), Charset.BASE64 (for
+ base64 encoding), or Charset.SHORTEST for the shortest of
QP or BASE64 encoding. Otherwise, it will be None.
body_encoding: Same as header_encoding, but describes the encoding for the
mail message's body, which indeed may be different than the
- header encoding. charset.SHORTEST is not allowed for
+ header encoding. Charset.SHORTEST is not allowed for
body_encoding.
output_charset: Some character sets must be converted before they can be
diff --git a/common/py3-stdlib/email/contentmanager.py b/common/py3-stdlib/email/contentmanager.py
index fcf278d..b4f5830 100644
--- a/common/py3-stdlib/email/contentmanager.py
+++ b/common/py3-stdlib/email/contentmanager.py
@@ -72,12 +72,14 @@
return msg.get_payload(decode=True)
for maintype in 'audio image video application'.split():
raw_data_manager.add_get_handler(maintype, get_non_text_content)
+del maintype
def get_message_content(msg):
return msg.get_payload(0)
for subtype in 'rfc822 external-body'.split():
raw_data_manager.add_get_handler('message/'+subtype, get_message_content)
+del subtype
def get_and_fixup_unknown_message_content(msg):
@@ -246,3 +248,4 @@
_finalize_set(msg, disposition, filename, cid, params)
for typ in (bytes, bytearray, memoryview):
raw_data_manager.add_set_handler(typ, set_bytes_content)
+del typ
diff --git a/common/py3-stdlib/email/headerregistry.py b/common/py3-stdlib/email/headerregistry.py
index b590d69..543141d 100644
--- a/common/py3-stdlib/email/headerregistry.py
+++ b/common/py3-stdlib/email/headerregistry.py
@@ -218,7 +218,7 @@
self.__class__.__bases__,
str(self),
),
- self.__dict__)
+ self.__getstate__())
@classmethod
def _reconstruct(cls, value):
diff --git a/common/py3-stdlib/email/message.py b/common/py3-stdlib/email/message.py
index 6752ce0..65fda50 100644
--- a/common/py3-stdlib/email/message.py
+++ b/common/py3-stdlib/email/message.py
@@ -6,8 +6,8 @@
__all__ = ['Message', 'EmailMessage']
+import binascii
import re
-import uu
import quopri
from io import BytesIO, StringIO
@@ -35,7 +35,7 @@
if not sep:
return a.strip(), None
return a.strip(), b.strip()
-
+
def _formatparam(param, value=None, quote=True):
"""Convenience function to format and return a key=value pair.
@@ -101,7 +101,37 @@
return utils.unquote(value)
-
+def _decode_uu(encoded):
+ """Decode uuencoded data."""
+ decoded_lines = []
+ encoded_lines_iter = iter(encoded.splitlines())
+ for line in encoded_lines_iter:
+ if line.startswith(b"begin "):
+ mode, _, path = line.removeprefix(b"begin ").partition(b" ")
+ try:
+ int(mode, base=8)
+ except ValueError:
+ continue
+ else:
+ break
+ else:
+ raise ValueError("`begin` line not found")
+ for line in encoded_lines_iter:
+ if not line:
+ raise ValueError("Truncated input")
+ elif line.strip(b' \t\r\n\f') == b'end':
+ break
+ try:
+ decoded_line = binascii.a2b_uu(line)
+ except binascii.Error:
+ # Workaround for broken uuencoders by /Fredrik Lundh
+ nbytes = (((line[0]-32) & 63) * 4 + 5) // 3
+ decoded_line = binascii.a2b_uu(line[:nbytes])
+ decoded_lines.append(decoded_line)
+
+ return b''.join(decoded_lines)
+
+
class Message:
"""Basic message object.
@@ -288,13 +318,10 @@
self.policy.handle_defect(self, defect)
return value
elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'):
- in_file = BytesIO(bpayload)
- out_file = BytesIO()
try:
- uu.decode(in_file, out_file, quiet=True)
- return out_file.getvalue()
- except uu.Error:
- # Some decoding problem
+ return _decode_uu(bpayload)
+ except ValueError:
+ # Some decoding problem.
return bpayload
if isinstance(payload, str):
return bpayload
diff --git a/common/py3-stdlib/email/mime/audio.py b/common/py3-stdlib/email/mime/audio.py
index 7cca3f9..065819b 100644
--- a/common/py3-stdlib/email/mime/audio.py
+++ b/common/py3-stdlib/email/mime/audio.py
@@ -6,39 +6,11 @@
__all__ = ['MIMEAudio']
-import sndhdr
-
from io import BytesIO
from email import encoders
from email.mime.nonmultipart import MIMENonMultipart
-
-_sndhdr_MIMEmap = {'au' : 'basic',
- 'wav' :'x-wav',
- 'aiff':'x-aiff',
- 'aifc':'x-aiff',
- }
-
-# There are others in sndhdr that don't have MIME types. :(
-# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma??
-def _whatsnd(data):
- """Try to identify a sound file type.
-
- sndhdr.what() has a pretty cruddy interface, unfortunately. This is why
- we re-do it here. It would be easier to reverse engineer the Unix 'file'
- command and use the standard 'magic' file, as shipped with a modern Unix.
- """
- hdr = data[:512]
- fakefile = BytesIO(hdr)
- for testfn in sndhdr.tests:
- res = testfn(hdr, fakefile)
- if res is not None:
- return _sndhdr_MIMEmap.get(res[0])
- return None
-
-
-
class MIMEAudio(MIMENonMultipart):
"""Class for generating audio/* MIME documents."""
@@ -47,7 +19,7 @@
"""Create an audio/* type MIME document.
_audiodata contains the bytes for the raw audio data. If this data
- can be decoded by the standard Python `sndhdr' module, then the
+ can be decoded as au, wav, aiff, or aifc, then the
subtype will be automatically included in the Content-Type header.
Otherwise, you can specify the specific audio subtype via the
_subtype parameter. If _subtype is not given, and no subtype can be
@@ -65,10 +37,64 @@
header.
"""
if _subtype is None:
- _subtype = _whatsnd(_audiodata)
+ _subtype = _what(_audiodata)
if _subtype is None:
raise TypeError('Could not find audio MIME subtype')
MIMENonMultipart.__init__(self, 'audio', _subtype, policy=policy,
**_params)
self.set_payload(_audiodata)
_encoder(self)
+
+
+_rules = []
+
+
+# Originally from the sndhdr module.
+#
+# There are others in sndhdr that don't have MIME types. :(
+# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma??
+def _what(data):
+ # Try to identify a sound file type.
+ #
+ # sndhdr.what() had a pretty cruddy interface, unfortunately. This is why
+ # we re-do it here. It would be easier to reverse engineer the Unix 'file'
+ # command and use the standard 'magic' file, as shipped with a modern Unix.
+ hdr = data[:512]
+ fakefile = BytesIO(hdr)
+ for testfn in _rules:
+ if res := testfn(hdr, fakefile):
+ return res
+ else:
+ return None
+
+
+def rule(rulefunc):
+ _rules.append(rulefunc)
+ return rulefunc
+
+
+@rule
+def _aiff(h, f):
+ if not h.startswith(b'FORM'):
+ return None
+ if h[8:12] in {b'AIFC', b'AIFF'}:
+ return 'x-aiff'
+ else:
+ return None
+
+
+@rule
+def _au(h, f):
+ if h.startswith(b'.snd'):
+ return 'basic'
+ else:
+ return None
+
+
+@rule
+def _wav(h, f):
+ # 'RIFF' <len> 'WAVE' 'fmt ' <len>
+ if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ':
+ return None
+ else:
+ return "x-wav"
diff --git a/common/py3-stdlib/email/mime/image.py b/common/py3-stdlib/email/mime/image.py
index 6dc7ec4..4b7f2f9 100644
--- a/common/py3-stdlib/email/mime/image.py
+++ b/common/py3-stdlib/email/mime/image.py
@@ -6,13 +6,10 @@
__all__ = ['MIMEImage']
-import imghdr
-
from email import encoders
from email.mime.nonmultipart import MIMENonMultipart
-
class MIMEImage(MIMENonMultipart):
"""Class for generating image/* type MIME documents."""
@@ -21,10 +18,10 @@
"""Create an image/* type MIME document.
_imagedata contains the bytes for the raw image data. If the data
- can be decoded by the standard Python `imghdr' module, then the
- subtype will be automatically included in the Content-Type header.
- Otherwise, you can specify the specific image subtype via the _subtype
- parameter.
+ type can be detected (jpeg, png, gif, tiff, rgb, pbm, pgm, ppm,
+ rast, xbm, bmp, webp, and exr attempted), then the subtype will be
+ automatically included in the Content-Type header. Otherwise, you can
+ specify the specific image subtype via the _subtype parameter.
_encoder is a function which will perform the actual encoding for
transport of the image data. It takes one argument, which is this
@@ -37,11 +34,119 @@
constructor, which turns them into parameters on the Content-Type
header.
"""
- if _subtype is None:
- _subtype = imghdr.what(None, _imagedata)
+ _subtype = _what(_imagedata) if _subtype is None else _subtype
if _subtype is None:
raise TypeError('Could not guess image MIME subtype')
MIMENonMultipart.__init__(self, 'image', _subtype, policy=policy,
**_params)
self.set_payload(_imagedata)
_encoder(self)
+
+
+_rules = []
+
+
+# Originally from the imghdr module.
+def _what(data):
+ for rule in _rules:
+ if res := rule(data):
+ return res
+ else:
+ return None
+
+
+def rule(rulefunc):
+ _rules.append(rulefunc)
+ return rulefunc
+
+
+@rule
+def _jpeg(h):
+ """JPEG data with JFIF or Exif markers; and raw JPEG"""
+ if h[6:10] in (b'JFIF', b'Exif'):
+ return 'jpeg'
+ elif h[:4] == b'\xff\xd8\xff\xdb':
+ return 'jpeg'
+
+
+@rule
+def _png(h):
+ if h.startswith(b'\211PNG\r\n\032\n'):
+ return 'png'
+
+
+@rule
+def _gif(h):
+ """GIF ('87 and '89 variants)"""
+ if h[:6] in (b'GIF87a', b'GIF89a'):
+ return 'gif'
+
+
+@rule
+def _tiff(h):
+ """TIFF (can be in Motorola or Intel byte order)"""
+ if h[:2] in (b'MM', b'II'):
+ return 'tiff'
+
+
+@rule
+def _rgb(h):
+ """SGI image library"""
+ if h.startswith(b'\001\332'):
+ return 'rgb'
+
+
+@rule
+def _pbm(h):
+ """PBM (portable bitmap)"""
+ if len(h) >= 3 and \
+ h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
+ return 'pbm'
+
+
+@rule
+def _pgm(h):
+ """PGM (portable graymap)"""
+ if len(h) >= 3 and \
+ h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
+ return 'pgm'
+
+
+@rule
+def _ppm(h):
+ """PPM (portable pixmap)"""
+ if len(h) >= 3 and \
+ h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
+ return 'ppm'
+
+
+@rule
+def _rast(h):
+ """Sun raster file"""
+ if h.startswith(b'\x59\xA6\x6A\x95'):
+ return 'rast'
+
+
+@rule
+def _xbm(h):
+ """X bitmap (X10 or X11)"""
+ if h.startswith(b'#define '):
+ return 'xbm'
+
+
+@rule
+def _bmp(h):
+ if h.startswith(b'BM'):
+ return 'bmp'
+
+
+@rule
+def _webp(h):
+ if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
+ return 'webp'
+
+
+@rule
+def _exr(h):
+ if h.startswith(b'\x76\x2f\x31\x01'):
+ return 'exr'
diff --git a/common/py3-stdlib/email/quoprimime.py b/common/py3-stdlib/email/quoprimime.py
index 94534f7..27fcbb5 100644
--- a/common/py3-stdlib/email/quoprimime.py
+++ b/common/py3-stdlib/email/quoprimime.py
@@ -148,6 +148,7 @@
_QUOPRI_BODY_ENCODE_MAP = _QUOPRI_BODY_MAP[:]
for c in b'\r\n':
_QUOPRI_BODY_ENCODE_MAP[c] = chr(c)
+del c
def body_encode(body, maxlinelen=76, eol=NL):
"""Encode with quoted-printable, wrapping at maxlinelen characters.
diff --git a/common/py3-stdlib/encodings/__init__.py b/common/py3-stdlib/encodings/__init__.py
index 4b37d33..f9075b8 100644
--- a/common/py3-stdlib/encodings/__init__.py
+++ b/common/py3-stdlib/encodings/__init__.py
@@ -156,6 +156,10 @@
codecs.register(search_function)
if sys.platform == 'win32':
+ # bpo-671666, bpo-46668: If Python does not implement a codec for current
+ # Windows ANSI code page, use the "mbcs" codec instead:
+ # WideCharToMultiByte() and MultiByteToWideChar() functions with CP_ACP.
+ # Python does not support custom code pages.
def _alias_mbcs(encoding):
try:
import _winapi
diff --git a/common/py3-stdlib/enum.py b/common/py3-stdlib/enum.py
index f5657a6..45e3cd0 100644
--- a/common/py3-stdlib/enum.py
+++ b/common/py3-stdlib/enum.py
@@ -1,14 +1,39 @@
import sys
+import builtins as bltns
from types import MappingProxyType, DynamicClassAttribute
+from operator import or_ as _or_
+from functools import reduce
__all__ = [
- 'EnumMeta',
- 'Enum', 'IntEnum', 'Flag', 'IntFlag',
- 'auto', 'unique',
+ 'EnumType', 'EnumMeta',
+ 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum',
+ 'auto', 'unique', 'property', 'verify', 'member', 'nonmember',
+ 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP',
+ 'global_flag_repr', 'global_enum_repr', 'global_str', 'global_enum',
+ 'EnumCheck', 'CONTINUOUS', 'NAMED_FLAGS', 'UNIQUE',
]
+# Dummy value for Enum and Flag as there are explicit checks for them
+# before they have been created.
+# This is also why there are checks in EnumType like `if Enum is not None`
+Enum = Flag = EJECT = _stdlib_enums = ReprEnum = None
+
+class nonmember(object):
+ """
+ Protects item from becoming an Enum member during class creation.
+ """
+ def __init__(self, value):
+ self.value = value
+
+class member(object):
+ """
+ Forces item to become an Enum member during class creation.
+ """
+ def __init__(self, value):
+ self.value = value
+
def _is_descriptor(obj):
"""
Returns True if obj is a descriptor, False otherwise.
@@ -41,6 +66,15 @@
name[-2:-1] != '_'
)
+def _is_internal_class(cls_name, obj):
+ # do not use `re` as `re` imports `enum`
+ if not isinstance(obj, type):
+ return False
+ qualname = getattr(obj, '__qualname__', '')
+ s_pattern = cls_name + '.' + getattr(obj, '__name__', '')
+ e_pattern = '.' + s_pattern
+ return qualname == s_pattern or qualname.endswith(e_pattern)
+
def _is_private(cls_name, name):
# do not use `re` as `re` imports `enum`
pattern = '_%s__' % (cls_name, )
@@ -55,33 +89,274 @@
else:
return False
-def _make_class_unpicklable(cls):
+def _is_single_bit(num):
"""
- Make the given class un-picklable.
+ True if only one bit set in num (should be an int)
+ """
+ if num == 0:
+ return False
+ num &= num - 1
+ return num == 0
+
+def _make_class_unpicklable(obj):
+ """
+ Make the given obj un-picklable.
+
+ obj should be either a dictionary, or an Enum
"""
def _break_on_call_reduce(self, proto):
raise TypeError('%r cannot be pickled' % self)
- cls.__reduce_ex__ = _break_on_call_reduce
- cls.__module__ = '<unknown>'
+ if isinstance(obj, dict):
+ obj['__reduce_ex__'] = _break_on_call_reduce
+ obj['__module__'] = '<unknown>'
+ else:
+ setattr(obj, '__reduce_ex__', _break_on_call_reduce)
+ setattr(obj, '__module__', '<unknown>')
-_auto_null = object()
+def _iter_bits_lsb(num):
+ # num must be a positive integer
+ original = num
+ if isinstance(num, Enum):
+ num = num.value
+ if num < 0:
+ raise ValueError('%r is not a positive integer' % original)
+ while num:
+ b = num & (~num + 1)
+ yield b
+ num ^= b
+
+def show_flag_values(value):
+ return list(_iter_bits_lsb(value))
+
+def bin(num, max_bits=None):
+ """
+ Like built-in bin(), except negative values are represented in
+ twos-compliment, and the leading bit always indicates sign
+ (0=positive, 1=negative).
+
+ >>> bin(10)
+ '0b0 1010'
+ >>> bin(~10) # ~10 is -11
+ '0b1 0101'
+ """
+
+ ceiling = 2 ** (num).bit_length()
+ if num >= 0:
+ s = bltns.bin(num + ceiling).replace('1', '0', 1)
+ else:
+ s = bltns.bin(~num ^ (ceiling - 1) + ceiling)
+ sign = s[:3]
+ digits = s[3:]
+ if max_bits is not None:
+ if len(digits) < max_bits:
+ digits = (sign[-1] * max_bits + digits)[-max_bits:]
+ return "%s %s" % (sign, digits)
+
+def _dedent(text):
+ """
+ Like textwrap.dedent. Rewritten because we cannot import textwrap.
+ """
+ lines = text.split('\n')
+ blanks = 0
+ for i, ch in enumerate(lines[0]):
+ if ch != ' ':
+ break
+ for j, l in enumerate(lines):
+ lines[j] = l[i:]
+ return '\n'.join(lines)
+
+class _auto_null:
+ def __repr__(self):
+ return '_auto_null'
+_auto_null = _auto_null()
+
class auto:
"""
Instances are replaced with an appropriate value in Enum class suites.
"""
- value = _auto_null
+ def __init__(self, value=_auto_null):
+ self.value = value
+
+ def __repr__(self):
+ return "auto(%r)" % self.value
+
+class property(DynamicClassAttribute):
+ """
+ This is a descriptor, used to define attributes that act differently
+ when accessed through an enum member and through an enum class.
+ Instance access is the same as property(), but access to an attribute
+ through the enum class will instead look in the class' _member_map_ for
+ a corresponding enum member.
+ """
+
+ def __get__(self, instance, ownerclass=None):
+ if instance is None:
+ try:
+ return ownerclass._member_map_[self.name]
+ except KeyError:
+ raise AttributeError(
+ '%r has no attribute %r' % (ownerclass, self.name)
+ )
+ else:
+ if self.fget is None:
+ # look for a member by this name.
+ try:
+ return ownerclass._member_map_[self.name]
+ except KeyError:
+ raise AttributeError(
+ '%r has no attribute %r' % (ownerclass, self.name)
+ ) from None
+ else:
+ return self.fget(instance)
+
+ def __set__(self, instance, value):
+ if self.fset is None:
+ raise AttributeError(
+ "<enum %r> cannot set attribute %r" % (self.clsname, self.name)
+ )
+ else:
+ return self.fset(instance, value)
+
+ def __delete__(self, instance):
+ if self.fdel is None:
+ raise AttributeError(
+ "<enum %r> cannot delete attribute %r" % (self.clsname, self.name)
+ )
+ else:
+ return self.fdel(instance)
+
+ def __set_name__(self, ownerclass, name):
+ self.name = name
+ self.clsname = ownerclass.__name__
+
+
+class _proto_member:
+ """
+ intermediate step for enum members between class execution and final creation
+ """
+
+ def __init__(self, value):
+ self.value = value
+
+ def __set_name__(self, enum_class, member_name):
+ """
+ convert each quasi-member into an instance of the new enum class
+ """
+ # first step: remove ourself from enum_class
+ delattr(enum_class, member_name)
+ # second step: create member based on enum_class
+ value = self.value
+ if not isinstance(value, tuple):
+ args = (value, )
+ else:
+ args = value
+ if enum_class._member_type_ is tuple: # special case for tuple enums
+ args = (args, ) # wrap it one more time
+ if not enum_class._use_args_:
+ enum_member = enum_class._new_member_(enum_class)
+ else:
+ enum_member = enum_class._new_member_(enum_class, *args)
+ if not hasattr(enum_member, '_value_'):
+ if enum_class._member_type_ is object:
+ enum_member._value_ = value
+ else:
+ try:
+ enum_member._value_ = enum_class._member_type_(*args)
+ except Exception as exc:
+ new_exc = TypeError(
+ '_value_ not set in __new__, unable to create it'
+ )
+ new_exc.__cause__ = exc
+ raise new_exc
+ value = enum_member._value_
+ enum_member._name_ = member_name
+ enum_member.__objclass__ = enum_class
+ enum_member.__init__(*args)
+ enum_member._sort_order_ = len(enum_class._member_names_)
+
+ if Flag is not None and issubclass(enum_class, Flag):
+ enum_class._flag_mask_ |= value
+ if _is_single_bit(value):
+ enum_class._singles_mask_ |= value
+ enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1
+
+ # If another member with the same value was already defined, the
+ # new member becomes an alias to the existing one.
+ try:
+ try:
+ # try to do a fast lookup to avoid the quadratic loop
+ enum_member = enum_class._value2member_map_[value]
+ except TypeError:
+ for name, canonical_member in enum_class._member_map_.items():
+ if canonical_member._value_ == value:
+ enum_member = canonical_member
+ break
+ else:
+ raise KeyError
+ except KeyError:
+ # this could still be an alias if the value is multi-bit and the
+ # class is a flag class
+ if (
+ Flag is None
+ or not issubclass(enum_class, Flag)
+ ):
+ # no other instances found, record this member in _member_names_
+ enum_class._member_names_.append(member_name)
+ elif (
+ Flag is not None
+ and issubclass(enum_class, Flag)
+ and _is_single_bit(value)
+ ):
+ # no other instances found, record this member in _member_names_
+ enum_class._member_names_.append(member_name)
+ # if necessary, get redirect in place and then add it to _member_map_
+ found_descriptor = None
+ for base in enum_class.__mro__[1:]:
+ descriptor = base.__dict__.get(member_name)
+ if descriptor is not None:
+ if isinstance(descriptor, (property, DynamicClassAttribute)):
+ found_descriptor = descriptor
+ break
+ elif (
+ hasattr(descriptor, 'fget') and
+ hasattr(descriptor, 'fset') and
+ hasattr(descriptor, 'fdel')
+ ):
+ found_descriptor = descriptor
+ continue
+ if found_descriptor:
+ redirect = property()
+ redirect.member = enum_member
+ redirect.__set_name__(enum_class, member_name)
+ # earlier descriptor found; copy fget, fset, fdel to this one.
+ redirect.fget = found_descriptor.fget
+ redirect.fset = found_descriptor.fset
+ redirect.fdel = found_descriptor.fdel
+ setattr(enum_class, member_name, redirect)
+ else:
+ setattr(enum_class, member_name, enum_member)
+ # now add to _member_map_ (even aliases)
+ enum_class._member_map_[member_name] = enum_member
+ try:
+ # This may fail if value is not hashable. We can't add the value
+ # to the map, and by-value lookups for this value will be
+ # linear.
+ enum_class._value2member_map_.setdefault(value, enum_member)
+ except TypeError:
+ # keep track of the value in a list so containment checks are quick
+ enum_class._unhashable_values_.append(value)
class _EnumDict(dict):
"""
Track enum member order and ensure member names are not reused.
- EnumMeta will use the names found in self._member_names as the
+ EnumType will use the names found in self._member_names as the
enumeration member names.
"""
def __init__(self):
super().__init__()
- self._member_names = []
+ self._member_names = {} # use a dict to keep insertion order
self._last_values = []
self._ignore = []
self._auto_called = False
@@ -95,25 +370,33 @@
Single underscore (sunder) names are reserved.
"""
- if _is_private(self._cls_name, key):
+ if _is_internal_class(self._cls_name, value):
import warnings
warnings.warn(
- "private variables, such as %r, will be normal attributes in 3.11"
- % (key, ),
+ "In 3.13 classes created inside an enum will not become a member. "
+ "Use the `member` decorator to keep the current behavior.",
DeprecationWarning,
stacklevel=2,
)
- if _is_sunder(key):
+ if _is_private(self._cls_name, key):
+ # also do nothing, name will be a normal attribute
+ pass
+ elif _is_sunder(key):
if key not in (
- '_order_', '_create_pseudo_member_',
- '_generate_next_value_', '_missing_', '_ignore_',
+ '_order_',
+ '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_',
+ '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_',
):
- raise ValueError('_names_ are reserved for future Enum use')
+ raise ValueError(
+ '_sunder_ names, such as %r, are reserved for future Enum use'
+ % (key, )
+ )
if key == '_generate_next_value_':
# check if members already defined as auto()
if self._auto_called:
raise TypeError("_generate_next_value_ must be defined before members")
- setattr(self, '_generate_next_value', value)
+ _gnv = value.__func__ if isinstance(value, staticmethod) else value
+ setattr(self, '_generate_next_value', _gnv)
elif key == '_ignore_':
if isinstance(value, str):
value = value.replace(',',' ').split()
@@ -131,41 +414,74 @@
key = '_order_'
elif key in self._member_names:
# descriptor overwriting an enum?
- raise TypeError('Attempted to reuse key: %r' % key)
+ raise TypeError('%r already defined as %r' % (key, self[key]))
elif key in self._ignore:
pass
- elif not _is_descriptor(value):
+ elif isinstance(value, nonmember):
+ # unwrap value here; it won't be processed by the below `else`
+ value = value.value
+ elif _is_descriptor(value):
+ pass
+ # TODO: uncomment next three lines in 3.13
+ # elif _is_internal_class(self._cls_name, value):
+ # # do nothing, name will be a normal attribute
+ # pass
+ else:
if key in self:
# enum overwriting a descriptor?
- raise TypeError('%r already defined as: %r' % (key, self[key]))
- if isinstance(value, auto):
- if value.value == _auto_null:
- value.value = self._generate_next_value(
- key,
- 1,
- len(self._member_names),
- self._last_values[:],
- )
- self._auto_called = True
+ raise TypeError('%r already defined as %r' % (key, self[key]))
+ elif isinstance(value, member):
+ # unwrap value here -- it will become a member
value = value.value
- self._member_names.append(key)
- self._last_values.append(value)
+ non_auto_store = True
+ single = False
+ if isinstance(value, auto):
+ single = True
+ value = (value, )
+ if type(value) is tuple and any(isinstance(v, auto) for v in value):
+ # insist on an actual tuple, no subclasses, in keeping with only supporting
+ # top-level auto() usage (not contained in any other data structure)
+ auto_valued = []
+ for v in value:
+ if isinstance(v, auto):
+ non_auto_store = False
+ if v.value == _auto_null:
+ v.value = self._generate_next_value(
+ key, 1, len(self._member_names), self._last_values[:],
+ )
+ self._auto_called = True
+ v = v.value
+ self._last_values.append(v)
+ auto_valued.append(v)
+ if single:
+ value = auto_valued[0]
+ else:
+ value = tuple(auto_valued)
+ self._member_names[key] = None
+ if non_auto_store:
+ self._last_values.append(value)
super().__setitem__(key, value)
+ def update(self, members, **more_members):
+ try:
+ for name in members.keys():
+ self[name] = members[name]
+ except AttributeError:
+ for name, value in members:
+ self[name] = value
+ for name, value in more_members.items():
+ self[name] = value
-# Dummy value for Enum as EnumMeta explicitly checks for it, but of course
-# until EnumMeta finishes running the first time the Enum class doesn't exist.
-# This is also why there are checks in EnumMeta like `if Enum is not None`
-Enum = None
-class EnumMeta(type):
+class EnumType(type):
"""
Metaclass for Enum
"""
+
@classmethod
def __prepare__(metacls, cls, bases, **kwds):
# check that previous enum members do not exist
- metacls._check_for_existing_members(cls, bases)
+ metacls._check_for_existing_members_(cls, bases)
# create the namespace dict
enum_dict = _EnumDict()
enum_dict._cls_name = cls
@@ -177,161 +493,125 @@
)
return enum_dict
- def __new__(metacls, cls, bases, classdict, **kwds):
+ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **kwds):
# an Enum class is final once enumeration items have been defined; it
# cannot be mixed with other types (int, float, etc.) if it has an
# inherited __new__ unless a new __new__ is defined (or the resulting
# class will fail).
#
+ if _simple:
+ return super().__new__(metacls, cls, bases, classdict, **kwds)
+ #
# remove any keys listed in _ignore_
classdict.setdefault('_ignore_', []).append('_ignore_')
ignore = classdict['_ignore_']
for key in ignore:
classdict.pop(key, None)
+ #
+ # grab member names
+ member_names = classdict._member_names
+ #
+ # check for illegal enum names (any others?)
+ invalid_names = set(member_names) & {'mro', ''}
+ if invalid_names:
+ raise ValueError('invalid enum member name(s) %s' % (
+ ','.join(repr(n) for n in invalid_names)
+ ))
+ #
+ # adjust the sunders
+ _order_ = classdict.pop('_order_', None)
+ # convert to normal dict
+ classdict = dict(classdict.items())
+ #
+ # data type of member and the controlling Enum class
member_type, first_enum = metacls._get_mixins_(cls, bases)
__new__, save_new, use_args = metacls._find_new_(
classdict, member_type, first_enum,
)
-
- # save enum items into separate mapping so they don't get baked into
- # the new class
- enum_members = {k: classdict[k] for k in classdict._member_names}
- for name in classdict._member_names:
- del classdict[name]
-
- # adjust the sunders
- _order_ = classdict.pop('_order_', None)
-
- # check for illegal enum names (any others?)
- invalid_names = set(enum_members) & {'mro', ''}
- if invalid_names:
- raise ValueError('Invalid enum member name: {0}'.format(
- ','.join(invalid_names)))
-
- # create a default docstring if one has not been provided
- if '__doc__' not in classdict:
- classdict['__doc__'] = 'An enumeration.'
-
- enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
- enum_class._member_names_ = [] # names in definition order
- enum_class._member_map_ = {} # name->value map
- enum_class._member_type_ = member_type
-
- # save DynamicClassAttribute attributes from super classes so we know
- # if we can take the shortcut of storing members in the class dict
- dynamic_attributes = {
- k for c in enum_class.mro()
- for k, v in c.__dict__.items()
- if isinstance(v, DynamicClassAttribute)
- }
-
- # Reverse value->name map for hashable values.
- enum_class._value2member_map_ = {}
-
- # If a custom type is mixed into the Enum, and it does not know how
- # to pickle itself, pickle.dumps will succeed but pickle.loads will
- # fail. Rather than have the error show up later and possibly far
- # from the source, sabotage the pickle protocol for this class so
- # that pickle.dumps also fails.
+ classdict['_new_member_'] = __new__
+ classdict['_use_args_'] = use_args
#
- # However, if the new class implements its own __reduce_ex__, do not
- # sabotage -- it's on them to make sure it works correctly. We use
- # __reduce_ex__ instead of any of the others as it is preferred by
- # pickle over __reduce__, and it handles all pickle protocols.
- if '__reduce_ex__' not in classdict:
- if member_type is not object:
- methods = ('__getnewargs_ex__', '__getnewargs__',
- '__reduce_ex__', '__reduce__')
- if not any(m in member_type.__dict__ for m in methods):
- if '__new__' in classdict:
- # too late, sabotage
- _make_class_unpicklable(enum_class)
- else:
- # final attempt to verify that pickling would work:
- # travel mro until __new__ is found, checking for
- # __reduce__ and friends along the way -- if any of them
- # are found before/when __new__ is found, pickling should
- # work
- sabotage = None
- for chain in bases:
- for base in chain.__mro__:
- if base is object:
- continue
- elif any(m in base.__dict__ for m in methods):
- # found one, we're good
- sabotage = False
- break
- elif '__new__' in base.__dict__:
- # not good
- sabotage = True
- break
- if sabotage is not None:
- break
- if sabotage:
- _make_class_unpicklable(enum_class)
- # instantiate them, checking for duplicates as we go
- # we instantiate first instead of checking for duplicates first in case
- # a custom __new__ is doing something funky with the values -- such as
- # auto-numbering ;)
- for member_name in classdict._member_names:
- value = enum_members[member_name]
- if not isinstance(value, tuple):
- args = (value, )
- else:
- args = value
- if member_type is tuple: # special case for tuple enums
- args = (args, ) # wrap it one more time
- if not use_args:
- enum_member = __new__(enum_class)
- if not hasattr(enum_member, '_value_'):
- enum_member._value_ = value
- else:
- enum_member = __new__(enum_class, *args)
- if not hasattr(enum_member, '_value_'):
- if member_type is object:
- enum_member._value_ = value
- else:
- enum_member._value_ = member_type(*args)
- value = enum_member._value_
- enum_member._name_ = member_name
- enum_member.__objclass__ = enum_class
- enum_member.__init__(*args)
- # If another member with the same value was already defined, the
- # new member becomes an alias to the existing one.
- for name, canonical_member in enum_class._member_map_.items():
- if canonical_member._value_ == enum_member._value_:
- enum_member = canonical_member
- break
- else:
- # Aliases don't appear in member names (only in __members__).
- enum_class._member_names_.append(member_name)
- # performance boost for any member that would not shadow
- # a DynamicClassAttribute
- if member_name not in dynamic_attributes:
- setattr(enum_class, member_name, enum_member)
- # now add to _member_map_
- enum_class._member_map_[member_name] = enum_member
- try:
- # This may fail if value is not hashable. We can't add the value
- # to the map, and by-value lookups for this value will be
- # linear.
- enum_class._value2member_map_[value] = enum_member
- except TypeError:
- pass
-
+ # convert future enum members into temporary _proto_members
+ for name in member_names:
+ value = classdict[name]
+ classdict[name] = _proto_member(value)
+ #
+ # house-keeping structures
+ classdict['_member_names_'] = []
+ classdict['_member_map_'] = {}
+ classdict['_value2member_map_'] = {}
+ classdict['_unhashable_values_'] = []
+ classdict['_member_type_'] = member_type
+ # now set the __repr__ for the value
+ classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases)
+ #
+ # Flag structures (will be removed if final class is not a Flag
+ classdict['_boundary_'] = (
+ boundary
+ or getattr(first_enum, '_boundary_', None)
+ )
+ classdict['_flag_mask_'] = 0
+ classdict['_singles_mask_'] = 0
+ classdict['_all_bits_'] = 0
+ classdict['_inverted_'] = None
+ try:
+ exc = None
+ enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
+ except RuntimeError as e:
+ # any exceptions raised by member.__new__ will get converted to a
+ # RuntimeError, so get that original exception back and raise it instead
+ exc = e.__cause__ or e
+ if exc is not None:
+ raise exc
+ #
+ # update classdict with any changes made by __init_subclass__
+ classdict.update(enum_class.__dict__)
+ #
# double check that repr and friends are not the mixin's or various
# things break (such as pickle)
# however, if the method is defined in the Enum itself, don't replace
# it
+ #
+ # Also, special handling for ReprEnum
+ if ReprEnum is not None and ReprEnum in bases:
+ if member_type is object:
+ raise TypeError(
+ 'ReprEnum subclasses must be mixed with a data type (i.e.'
+ ' int, str, float, etc.)'
+ )
+ if '__format__' not in classdict:
+ enum_class.__format__ = member_type.__format__
+ classdict['__format__'] = enum_class.__format__
+ if '__str__' not in classdict:
+ method = member_type.__str__
+ if method is object.__str__:
+ # if member_type does not define __str__, object.__str__ will use
+ # its __repr__ instead, so we'll also use its __repr__
+ method = member_type.__repr__
+ enum_class.__str__ = method
+ classdict['__str__'] = enum_class.__str__
for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'):
- if name in classdict:
- continue
- class_method = getattr(enum_class, name)
- obj_method = getattr(member_type, name, None)
- enum_method = getattr(first_enum, name, None)
- if obj_method is not None and obj_method is class_method:
- setattr(enum_class, name, enum_method)
-
+ if name not in classdict:
+ # check for mixin overrides before replacing
+ enum_method = getattr(first_enum, name)
+ found_method = getattr(enum_class, name)
+ object_method = getattr(object, name)
+ data_type_method = getattr(member_type, name)
+ if found_method in (data_type_method, object_method):
+ setattr(enum_class, name, enum_method)
+ #
+ # for Flag, add __or__, __and__, __xor__, and __invert__
+ if Flag is not None and issubclass(enum_class, Flag):
+ for name in (
+ '__or__', '__and__', '__xor__',
+ '__ror__', '__rand__', '__rxor__',
+ '__invert__'
+ ):
+ if name not in classdict:
+ enum_method = getattr(Flag, name)
+ setattr(enum_class, name, enum_method)
+ classdict[name] = enum_method
+ #
# replace any other __new__ with our own (as long as Enum is not None,
# anyway) -- again, this is to support pickle
if Enum is not None:
@@ -340,23 +620,69 @@
if save_new:
enum_class.__new_member__ = __new__
enum_class.__new__ = Enum.__new__
-
+ #
# py3 support for definition order (helps keep py2/py3 code in sync)
+ #
+ # _order_ checking is spread out into three/four steps
+ # - if enum_class is a Flag:
+ # - remove any non-single-bit flags from _order_
+ # - remove any aliases from _order_
+ # - check that _order_ and _member_names_ match
+ #
+ # step 1: ensure we have a list
if _order_ is not None:
if isinstance(_order_, str):
_order_ = _order_.replace(',', ' ').split()
+ #
+ # remove Flag structures if final class is not a Flag
+ if (
+ Flag is None and cls != 'Flag'
+ or Flag is not None and not issubclass(enum_class, Flag)
+ ):
+ delattr(enum_class, '_boundary_')
+ delattr(enum_class, '_flag_mask_')
+ delattr(enum_class, '_singles_mask_')
+ delattr(enum_class, '_all_bits_')
+ delattr(enum_class, '_inverted_')
+ elif Flag is not None and issubclass(enum_class, Flag):
+ # set correct __iter__
+ member_list = [m._value_ for m in enum_class]
+ if member_list != sorted(member_list):
+ enum_class._iter_member_ = enum_class._iter_member_by_def_
+ if _order_:
+ # _order_ step 2: remove any items from _order_ that are not single-bit
+ _order_ = [
+ o
+ for o in _order_
+ if o not in enum_class._member_map_ or _is_single_bit(enum_class[o]._value_)
+ ]
+ #
+ if _order_:
+ # _order_ step 3: remove aliases from _order_
+ _order_ = [
+ o
+ for o in _order_
+ if (
+ o not in enum_class._member_map_
+ or
+ (o in enum_class._member_map_ and o in enum_class._member_names_)
+ )]
+ # _order_ step 4: verify that _order_ and _member_names_ match
if _order_ != enum_class._member_names_:
- raise TypeError('member order does not match _order_')
+ raise TypeError(
+ 'member order does not match _order_:\n %r\n %r'
+ % (enum_class._member_names_, _order_)
+ )
return enum_class
- def __bool__(self):
+ def __bool__(cls):
"""
classes/types should always be True.
"""
return True
- def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):
+ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None):
"""
Either returns an existing member, or creates a new enum class.
@@ -391,34 +717,54 @@
qualname=qualname,
type=type,
start=start,
+ boundary=boundary,
)
- def __contains__(cls, obj):
- if not isinstance(obj, Enum):
+ def __contains__(cls, member):
+ """
+ Return True if member is a member of this enum
+ raises TypeError if member is not an enum member
+
+ note: in 3.12 TypeError will no longer be raised, and True will also be
+ returned if member is the value of a member in this enum
+ """
+ if not isinstance(member, Enum):
import warnings
warnings.warn(
- "in 3.12 __contains__ will no longer raise TypeError, but will return True if\n"
- "obj is a member or a member's value",
+ "in 3.12 __contains__ will no longer raise TypeError, but will return True or\n"
+ "False depending on whether the value is a member or the value of a member",
DeprecationWarning,
stacklevel=2,
)
raise TypeError(
"unsupported operand type(s) for 'in': '%s' and '%s'" % (
- type(obj).__qualname__, cls.__class__.__qualname__))
- return isinstance(obj, cls) and obj._name_ in cls._member_map_
+ type(member).__qualname__, cls.__class__.__qualname__))
+ return isinstance(member, cls) and member._name_ in cls._member_map_
def __delattr__(cls, attr):
# nicer error message when someone tries to delete an attribute
# (see issue19025).
if attr in cls._member_map_:
- raise AttributeError("%s: cannot delete Enum member." % cls.__name__)
+ raise AttributeError("%r cannot delete member %r." % (cls.__name__, attr))
super().__delattr__(attr)
- def __dir__(self):
- return (
- ['__class__', '__doc__', '__members__', '__module__']
- + self._member_names_
+ def __dir__(cls):
+ interesting = set([
+ '__class__', '__contains__', '__doc__', '__getitem__',
+ '__iter__', '__len__', '__members__', '__module__',
+ '__name__', '__qualname__',
+ ]
+ + cls._member_names_
)
+ if cls._new_member_ is not object.__new__:
+ interesting.add('__new__')
+ if cls.__init_subclass__ is not object.__init_subclass__:
+ interesting.add('__init_subclass__')
+ if cls._member_type_ is object:
+ return sorted(interesting)
+ else:
+ # return whatever mixed-in data type has
+ return sorted(set(dir(cls._member_type_)) | interesting)
def __getattr__(cls, name):
"""
@@ -437,18 +783,24 @@
raise AttributeError(name) from None
def __getitem__(cls, name):
+ """
+ Return the member matching `name`.
+ """
return cls._member_map_[name]
def __iter__(cls):
"""
- Returns members in definition order.
+ Return members in definition order.
"""
return (cls._member_map_[name] for name in cls._member_names_)
def __len__(cls):
+ """
+ Return the number of members (no aliases)
+ """
return len(cls._member_names_)
- @property
+ @bltns.property
def __members__(cls):
"""
Returns a mapping of member name->value.
@@ -459,11 +811,14 @@
return MappingProxyType(cls._member_map_)
def __repr__(cls):
- return "<enum %r>" % cls.__name__
+ if Flag is not None and issubclass(cls, Flag):
+ return "<flag %r>" % cls.__name__
+ else:
+ return "<enum %r>" % cls.__name__
def __reversed__(cls):
"""
- Returns members in reverse definition order.
+ Return members in reverse definition order.
"""
return (cls._member_map_[name] for name in reversed(cls._member_names_))
@@ -477,10 +832,10 @@
"""
member_map = cls.__dict__.get('_member_map_', {})
if name in member_map:
- raise AttributeError('Cannot reassign members.')
+ raise AttributeError('cannot reassign member %r' % (name, ))
super().__setattr__(name, value)
- def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1):
+ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1, boundary=None):
"""
Convenience method to create a new Enum class.
@@ -494,7 +849,7 @@
"""
metacls = cls.__class__
bases = (cls, ) if type is None else (type, cls)
- _, first_enum = cls._get_mixins_(cls, bases)
+ _, first_enum = cls._get_mixins_(class_name, bases)
classdict = metacls.__prepare__(class_name, bases)
# special processing needed for names?
@@ -515,7 +870,6 @@
else:
member_name, member_value = item
classdict[member_name] = member_value
- enum_class = metacls.__new__(metacls, class_name, bases, classdict)
# TODO: replace the frame hack if a blessed way to know the calling
# module is ever developed
@@ -525,15 +879,15 @@
except (AttributeError, ValueError, KeyError):
pass
if module is None:
- _make_class_unpicklable(enum_class)
+ _make_class_unpicklable(classdict)
else:
- enum_class.__module__ = module
+ classdict['__module__'] = module
if qualname is not None:
- enum_class.__qualname__ = qualname
+ classdict['__qualname__'] = qualname
- return enum_class
+ return metacls.__new__(metacls, class_name, bases, classdict, boundary=boundary)
- def _convert_(cls, name, module, filter, source=None):
+ def _convert_(cls, name, module, filter, source=None, *, boundary=None, as_global=False):
"""
Create a new Enum subclass that replaces a collection of global constants
"""
@@ -542,9 +896,9 @@
# module;
# also, replace the __reduce_ex__ method so unpickling works in
# previous Python versions
- module_globals = vars(sys.modules[module])
+ module_globals = sys.modules[module].__dict__
if source:
- source = vars(source)
+ source = source.__dict__
else:
source = module_globals
# _value2member_map_ is populated in the same order every time
@@ -560,24 +914,30 @@
except TypeError:
# unless some values aren't comparable, in which case sort by name
members.sort(key=lambda t: t[0])
- cls = cls(name, members, module=module)
- cls.__reduce_ex__ = _reduce_ex_by_name
- module_globals.update(cls.__members__)
+ body = {t[0]: t[1] for t in members}
+ body['__module__'] = module
+ tmp_cls = type(name, (object, ), body)
+ cls = _simple_enum(etype=cls, boundary=boundary or KEEP)(tmp_cls)
+ cls.__reduce_ex__ = _reduce_ex_by_global_name
+ if as_global:
+ global_enum(cls)
+ else:
+ sys.modules[cls.__module__].__dict__.update(cls.__members__)
module_globals[name] = cls
return cls
- @staticmethod
- def _check_for_existing_members(class_name, bases):
+ @classmethod
+ def _check_for_existing_members_(mcls, class_name, bases):
for chain in bases:
for base in chain.__mro__:
- if issubclass(base, Enum) and base._member_names_:
+ if isinstance(base, EnumType) and base._member_names_:
raise TypeError(
- "%s: cannot extend enumeration %r"
- % (class_name, base.__name__)
+ "<enum %r> cannot extend %r"
+ % (class_name, base)
)
- @staticmethod
- def _get_mixins_(class_name, bases):
+ @classmethod
+ def _get_mixins_(mcls, class_name, bases):
"""
Returns the type for creating enum members, and the first inherited
enum class.
@@ -587,44 +947,62 @@
if not bases:
return object, Enum
- def _find_data_type(bases):
- data_types = set()
- for chain in bases:
- candidate = None
- for base in chain.__mro__:
- if base is object:
- continue
- elif issubclass(base, Enum):
- if base._member_type_ is not object:
- data_types.add(base._member_type_)
- break
- elif '__new__' in base.__dict__:
- if issubclass(base, Enum):
- continue
- data_types.add(candidate or base)
- break
- else:
- candidate = candidate or base
- if len(data_types) > 1:
- raise TypeError('%r: too many data types: %r' % (class_name, data_types))
- elif data_types:
- return data_types.pop()
- else:
- return None
+ mcls._check_for_existing_members_(class_name, bases)
# ensure final parent class is an Enum derivative, find any concrete
# data type, and check that Enum has no members
first_enum = bases[-1]
- if not issubclass(first_enum, Enum):
+ if not isinstance(first_enum, EnumType):
raise TypeError("new enumerations should be created as "
"`EnumName([mixin_type, ...] [data_type,] enum_type)`")
- member_type = _find_data_type(bases) or object
- if first_enum._member_names_:
- raise TypeError("Cannot extend enumerations")
+ member_type = mcls._find_data_type_(class_name, bases) or object
return member_type, first_enum
- @staticmethod
- def _find_new_(classdict, member_type, first_enum):
+ @classmethod
+ def _find_data_repr_(mcls, class_name, bases):
+ for chain in bases:
+ for base in chain.__mro__:
+ if base is object:
+ continue
+ elif isinstance(base, EnumType):
+ # if we hit an Enum, use it's _value_repr_
+ return base._value_repr_
+ elif '__repr__' in base.__dict__:
+ # this is our data repr
+ return base.__dict__['__repr__']
+ return None
+
+ @classmethod
+ def _find_data_type_(mcls, class_name, bases):
+ # a datatype has a __new__ method
+ data_types = set()
+ base_chain = set()
+ for chain in bases:
+ candidate = None
+ for base in chain.__mro__:
+ base_chain.add(base)
+ if base is object:
+ continue
+ elif isinstance(base, EnumType):
+ if base._member_type_ is not object:
+ data_types.add(base._member_type_)
+ break
+ elif '__new__' in base.__dict__ or '__dataclass_fields__' in base.__dict__:
+ if isinstance(base, EnumType):
+ continue
+ data_types.add(candidate or base)
+ break
+ else:
+ candidate = candidate or base
+ if len(data_types) > 1:
+ raise TypeError('too many data types for %r: %r' % (class_name, data_types))
+ elif data_types:
+ return data_types.pop()
+ else:
+ return None
+
+ @classmethod
+ def _find_new_(mcls, classdict, member_type, first_enum):
"""
Returns the __new__ to be used for creating the enum members.
@@ -638,7 +1016,7 @@
__new__ = classdict.get('__new__', None)
# should __new__ be saved as __new_member__ later?
- save_new = __new__ is not None
+ save_new = first_enum is not None and __new__ is not None
if __new__ is None:
# check all possibles for __new_member__ before falling back to
@@ -662,19 +1040,54 @@
# if a non-object.__new__ is used then whatever value/tuple was
# assigned to the enum member name will be passed to __new__ and to the
# new enum member's __init__
- if __new__ is object.__new__:
+ if first_enum is None or __new__ in (Enum.__new__, object.__new__):
use_args = False
else:
use_args = True
return __new__, save_new, use_args
+EnumMeta = EnumType
-class Enum(metaclass=EnumMeta):
+class Enum(metaclass=EnumType):
"""
- Generic enumeration.
+ Create a collection of name/value pairs.
- Derive from this class to define new enumerations.
+ Example enumeration:
+
+ >>> class Color(Enum):
+ ... RED = 1
+ ... BLUE = 2
+ ... GREEN = 3
+
+ Access them by:
+
+ - attribute access::
+
+ >>> Color.RED
+ <Color.RED: 1>
+
+ - value lookup:
+
+ >>> Color(1)
+ <Color.RED: 1>
+
+ - name lookup:
+
+ >>> Color['RED']
+ <Color.RED: 1>
+
+ Enumerations can be iterated over, and know how many members they have:
+
+ >>> len(Color)
+ 3
+
+ >>> list(Color)
+ [<Color.RED: 1>, <Color.BLUE: 2>, <Color.GREEN: 3>]
+
+ Methods can be added to enumerations, and members can have their own
+ attributes -- see the documentation for details.
"""
+
def __new__(cls, value):
# all enum instances are actually created during class construction
# without calling this method; this method is called by the metaclass'
@@ -704,6 +1117,11 @@
try:
if isinstance(result, cls):
return result
+ elif (
+ Flag is not None and issubclass(cls, Flag)
+ and cls._boundary_ is EJECT and isinstance(result, int)
+ ):
+ return result
else:
ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__))
if result is None and exc is None:
@@ -721,6 +1139,9 @@
exc = None
ve_exc = None
+ def __init__(self, *args, **kwds):
+ pass
+
def _generate_next_value_(name, start, count, last_values):
"""
Generate the next value when not given.
@@ -728,14 +1149,32 @@
name: the name of the member
start: the initial start value or None
count: the number of existing members
- last_value: the last value assigned or None
+ last_values: the list of values assigned
"""
- for last_value in reversed(last_values):
- try:
- return last_value + 1
- except TypeError:
- pass
- else:
+ if not last_values:
+ return start
+ try:
+ last = last_values[-1]
+ last_values.sort()
+ if last == last_values[-1]:
+ # no difference between old and new methods
+ return last + 1
+ else:
+ # trigger old method (with warning)
+ raise TypeError
+ except TypeError:
+ import warnings
+ warnings.warn(
+ "In 3.13 the default `auto()`/`_generate_next_value_` will require all values to be sortable and support adding +1\n"
+ "and the value returned will be the largest value in the enum incremented by 1",
+ DeprecationWarning,
+ stacklevel=3,
+ )
+ for v in reversed(last_values):
+ try:
+ return v + 1
+ except TypeError:
+ pass
return start
@classmethod
@@ -743,79 +1182,157 @@
return None
def __repr__(self):
- return "<%s.%s: %r>" % (
- self.__class__.__name__, self._name_, self._value_)
+ v_repr = self.__class__._value_repr_ or repr
+ return "<%s.%s: %s>" % (self.__class__.__name__, self._name_, v_repr(self._value_))
def __str__(self):
- return "%s.%s" % (self.__class__.__name__, self._name_)
+ return "%s.%s" % (self.__class__.__name__, self._name_, )
def __dir__(self):
"""
Returns all members and all public methods
"""
- added_behavior = [
- m
- for cls in self.__class__.mro()
- for m in cls.__dict__
- if m[0] != '_' and m not in self._member_map_
- ] + [m for m in self.__dict__ if m[0] != '_']
- return (['__class__', '__doc__', '__module__'] + added_behavior)
+ if self.__class__._member_type_ is object:
+ interesting = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value'])
+ else:
+ interesting = set(object.__dir__(self))
+ for name in getattr(self, '__dict__', []):
+ if name[0] != '_':
+ interesting.add(name)
+ for cls in self.__class__.mro():
+ for name, obj in cls.__dict__.items():
+ if name[0] == '_':
+ continue
+ if isinstance(obj, property):
+ # that's an enum.property
+ if obj.fget is not None or name not in self._member_map_:
+ interesting.add(name)
+ else:
+ # in case it was added by `dir(self)`
+ interesting.discard(name)
+ else:
+ interesting.add(name)
+ names = sorted(
+ set(['__class__', '__doc__', '__eq__', '__hash__', '__module__'])
+ | interesting
+ )
+ return names
def __format__(self, format_spec):
- """
- Returns format using actual value type unless __str__ has been overridden.
- """
- # mixed-in Enums should use the mixed-in type's __format__, otherwise
- # we can get strange results with the Enum name showing up instead of
- # the value
-
- # pure Enum branch, or branch with __str__ explicitly overridden
- str_overridden = type(self).__str__ not in (Enum.__str__, Flag.__str__)
- if self._member_type_ is object or str_overridden:
- cls = str
- val = str(self)
- # mix-in branch
- else:
- cls = self._member_type_
- val = self._value_
- return cls.__format__(val, format_spec)
+ return str.__format__(str(self), format_spec)
def __hash__(self):
return hash(self._name_)
def __reduce_ex__(self, proto):
- return self.__class__, (self._value_, )
+ return getattr, (self.__class__, self._name_)
- # DynamicClassAttribute is used to provide access to the `name` and
- # `value` properties of enum members while keeping some measure of
+ # enum.property is used to provide access to the `name` and
+ # `value` attributes of enum members while keeping some measure of
# protection from modification, while still allowing for an enumeration
# to have members named `name` and `value`. This works because enumeration
- # members are not set directly on the enum class -- __getattr__ is
- # used to look them up.
+ # members are not set directly on the enum class; they are kept in a
+ # separate structure, _member_map_, which is where enum.property looks for
+ # them
- @DynamicClassAttribute
+ @property
def name(self):
"""The name of the Enum member."""
return self._name_
- @DynamicClassAttribute
+ @property
def value(self):
"""The value of the Enum member."""
return self._value_
-class IntEnum(int, Enum):
- """Enum where members are also (and must be) ints"""
+class ReprEnum(Enum):
+ """
+ Only changes the repr(), leaving str() and format() to the mixed-in type.
+ """
-def _reduce_ex_by_name(self, proto):
+class IntEnum(int, ReprEnum):
+ """
+ Enum where members are also (and must be) ints
+ """
+
+
+class StrEnum(str, ReprEnum):
+ """
+ Enum where members are also (and must be) strings
+ """
+
+ def __new__(cls, *values):
+ "values must already be of type `str`"
+ if len(values) > 3:
+ raise TypeError('too many arguments for str(): %r' % (values, ))
+ if len(values) == 1:
+ # it must be a string
+ if not isinstance(values[0], str):
+ raise TypeError('%r is not a string' % (values[0], ))
+ if len(values) >= 2:
+ # check that encoding argument is a string
+ if not isinstance(values[1], str):
+ raise TypeError('encoding must be a string, not %r' % (values[1], ))
+ if len(values) == 3:
+ # check that errors argument is a string
+ if not isinstance(values[2], str):
+ raise TypeError('errors must be a string, not %r' % (values[2]))
+ value = str(*values)
+ member = str.__new__(cls, value)
+ member._value_ = value
+ return member
+
+ def _generate_next_value_(name, start, count, last_values):
+ """
+ Return the lower-cased version of the member name.
+ """
+ return name.lower()
+
+
+def _reduce_ex_by_global_name(self, proto):
return self.name
-class Flag(Enum):
+class FlagBoundary(StrEnum):
+ """
+ control how out of range values are handled
+ "strict" -> error is raised [default for Flag]
+ "conform" -> extra bits are discarded
+ "eject" -> lose flag status
+ "keep" -> keep flag status and all bits [default for IntFlag]
+ """
+ STRICT = auto()
+ CONFORM = auto()
+ EJECT = auto()
+ KEEP = auto()
+STRICT, CONFORM, EJECT, KEEP = FlagBoundary
+
+
+class Flag(Enum, boundary=STRICT):
"""
Support for flags
"""
+ def __reduce_ex__(self, proto):
+ cls = self.__class__
+ unknown = self._value_ & ~cls._flag_mask_
+ member_value = self._value_ & cls._flag_mask_
+ if unknown and member_value:
+ return _or_, (cls(member_value), unknown)
+ for val in _iter_bits_lsb(member_value):
+ rest = member_value & ~val
+ if rest:
+ return _or_, (cls(rest), cls._value2member_map_.get(val))
+ else:
+ break
+ if self._name_ is None:
+ return cls, (self._value_,)
+ else:
+ return getattr, (cls, self._name_)
+
+ _numeric_repr_ = repr
+
def _generate_next_value_(name, start, count, last_values):
"""
Generate the next value when not given.
@@ -823,49 +1340,129 @@
name: the name of the member
start: the initial start value or None
count: the number of existing members
- last_value: the last value assigned or None
+ last_values: the last value assigned or None
"""
if not count:
return start if start is not None else 1
- for last_value in reversed(last_values):
- try:
- high_bit = _high_bit(last_value)
- break
- except Exception:
- raise TypeError('Invalid Flag value: %r' % last_value) from None
+ last_value = max(last_values)
+ try:
+ high_bit = _high_bit(last_value)
+ except Exception:
+ raise TypeError('invalid flag value %r' % last_value) from None
return 2 ** (high_bit+1)
@classmethod
+ def _iter_member_by_value_(cls, value):
+ """
+ Extract all members from the value in definition (i.e. increasing value) order.
+ """
+ for val in _iter_bits_lsb(value & cls._flag_mask_):
+ yield cls._value2member_map_.get(val)
+
+ _iter_member_ = _iter_member_by_value_
+
+ @classmethod
+ def _iter_member_by_def_(cls, value):
+ """
+ Extract all members from the value in definition order.
+ """
+ yield from sorted(
+ cls._iter_member_by_value_(value),
+ key=lambda m: m._sort_order_,
+ )
+
+ @classmethod
def _missing_(cls, value):
"""
- Returns member (possibly creating it) if one can be found for value.
- """
- original_value = value
- if value < 0:
- value = ~value
- possible_member = cls._create_pseudo_member_(value)
- if original_value < 0:
- possible_member = ~possible_member
- return possible_member
+ Create a composite member containing all canonical members present in `value`.
- @classmethod
- def _create_pseudo_member_(cls, value):
+ If non-member values are present, result depends on `_boundary_` setting.
"""
- Create a composite member iff value contains only members.
- """
- pseudo_member = cls._value2member_map_.get(value, None)
- if pseudo_member is None:
- # verify all bits are accounted for
- _, extra_flags = _decompose(cls, value)
- if extra_flags:
- raise ValueError("%r is not a valid %s" % (value, cls.__qualname__))
+ if not isinstance(value, int):
+ raise ValueError(
+ "%r is not a valid %s" % (value, cls.__qualname__)
+ )
+ # check boundaries
+ # - value must be in range (e.g. -16 <-> +15, i.e. ~15 <-> 15)
+ # - value must not include any skipped flags (e.g. if bit 2 is not
+ # defined, then 0d10 is invalid)
+ flag_mask = cls._flag_mask_
+ singles_mask = cls._singles_mask_
+ all_bits = cls._all_bits_
+ neg_value = None
+ if (
+ not ~all_bits <= value <= all_bits
+ or value & (all_bits ^ flag_mask)
+ ):
+ if cls._boundary_ is STRICT:
+ max_bits = max(value.bit_length(), flag_mask.bit_length())
+ raise ValueError(
+ "%r invalid value %r\n given %s\n allowed %s" % (
+ cls, value, bin(value, max_bits), bin(flag_mask, max_bits),
+ ))
+ elif cls._boundary_ is CONFORM:
+ value = value & flag_mask
+ elif cls._boundary_ is EJECT:
+ return value
+ elif cls._boundary_ is KEEP:
+ if value < 0:
+ value = (
+ max(all_bits+1, 2**(value.bit_length()))
+ + value
+ )
+ else:
+ raise ValueError(
+ '%r unknown flag boundary %r' % (cls, cls._boundary_, )
+ )
+ if value < 0:
+ neg_value = value
+ value = all_bits + 1 + value
+ # get members and unknown
+ unknown = value & ~flag_mask
+ aliases = value & ~singles_mask
+ member_value = value & singles_mask
+ if unknown and cls._boundary_ is not KEEP:
+ raise ValueError(
+ '%s(%r) --> unknown values %r [%s]'
+ % (cls.__name__, value, unknown, bin(unknown))
+ )
+ # normal Flag?
+ if cls._member_type_ is object:
# construct a singleton enum pseudo-member
pseudo_member = object.__new__(cls)
- pseudo_member._name_ = None
+ else:
+ pseudo_member = cls._member_type_.__new__(cls, value)
+ if not hasattr(pseudo_member, '_value_'):
pseudo_member._value_ = value
- # use setdefault in case another thread already created a composite
- # with this value
+ if member_value or aliases:
+ members = []
+ combined_value = 0
+ for m in cls._iter_member_(member_value):
+ members.append(m)
+ combined_value |= m._value_
+ if aliases:
+ value = member_value | aliases
+ for n, pm in cls._member_map_.items():
+ if pm not in members and pm._value_ and pm._value_ & value == pm._value_:
+ members.append(pm)
+ combined_value |= pm._value_
+ unknown = value ^ combined_value
+ pseudo_member._name_ = '|'.join([m._name_ for m in members])
+ if not combined_value:
+ pseudo_member._name_ = None
+ elif unknown and cls._boundary_ is STRICT:
+ raise ValueError('%r: no members with value %r' % (cls, unknown))
+ elif unknown:
+ pseudo_member._name_ += '|%s' % cls._numeric_repr_(unknown)
+ else:
+ pseudo_member._name_ = None
+ # use setdefault in case another thread already created a composite
+ # with this value, but only if all members are known
+ # note: zero is a special case -- add it
+ if not unknown:
pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member)
+ if neg_value is not None:
+ cls._value2member_map_[neg_value] = pseudo_member
return pseudo_member
def __contains__(self, other):
@@ -874,133 +1471,89 @@
"""
if not isinstance(other, self.__class__):
raise TypeError(
- "unsupported operand type(s) for 'in': '%s' and '%s'" % (
+ "unsupported operand type(s) for 'in': %r and %r" % (
type(other).__qualname__, self.__class__.__qualname__))
return other._value_ & self._value_ == other._value_
+ def __iter__(self):
+ """
+ Returns flags in definition order.
+ """
+ yield from self._iter_member_(self._value_)
+
+ def __len__(self):
+ return self._value_.bit_count()
+
def __repr__(self):
- cls = self.__class__
- if self._name_ is not None:
- return '<%s.%s: %r>' % (cls.__name__, self._name_, self._value_)
- members, uncovered = _decompose(cls, self._value_)
- return '<%s.%s: %r>' % (
- cls.__name__,
- '|'.join([str(m._name_ or m._value_) for m in members]),
- self._value_,
- )
+ cls_name = self.__class__.__name__
+ v_repr = self.__class__._value_repr_ or repr
+ if self._name_ is None:
+ return "<%s: %s>" % (cls_name, v_repr(self._value_))
+ else:
+ return "<%s.%s: %s>" % (cls_name, self._name_, v_repr(self._value_))
def __str__(self):
- cls = self.__class__
- if self._name_ is not None:
- return '%s.%s' % (cls.__name__, self._name_)
- members, uncovered = _decompose(cls, self._value_)
- if len(members) == 1 and members[0]._name_ is None:
- return '%s.%r' % (cls.__name__, members[0]._value_)
+ cls_name = self.__class__.__name__
+ if self._name_ is None:
+ return '%s(%r)' % (cls_name, self._value_)
else:
- return '%s.%s' % (
- cls.__name__,
- '|'.join([str(m._name_ or m._value_) for m in members]),
- )
+ return "%s.%s" % (cls_name, self._name_)
def __bool__(self):
return bool(self._value_)
def __or__(self, other):
- if not isinstance(other, self.__class__):
+ if isinstance(other, self.__class__):
+ other = other._value_
+ elif self._member_type_ is not object and isinstance(other, self._member_type_):
+ other = other
+ else:
return NotImplemented
- return self.__class__(self._value_ | other._value_)
+ value = self._value_
+ return self.__class__(value | other)
def __and__(self, other):
- if not isinstance(other, self.__class__):
+ if isinstance(other, self.__class__):
+ other = other._value_
+ elif self._member_type_ is not object and isinstance(other, self._member_type_):
+ other = other
+ else:
return NotImplemented
- return self.__class__(self._value_ & other._value_)
+ value = self._value_
+ return self.__class__(value & other)
def __xor__(self, other):
- if not isinstance(other, self.__class__):
+ if isinstance(other, self.__class__):
+ other = other._value_
+ elif self._member_type_ is not object and isinstance(other, self._member_type_):
+ other = other
+ else:
return NotImplemented
- return self.__class__(self._value_ ^ other._value_)
+ value = self._value_
+ return self.__class__(value ^ other)
def __invert__(self):
- members, uncovered = _decompose(self.__class__, self._value_)
- inverted = self.__class__(0)
- for m in self.__class__:
- if m not in members and not (m._value_ & self._value_):
- inverted = inverted | m
- return self.__class__(inverted)
+ if self._inverted_ is None:
+ if self._boundary_ is KEEP:
+ # use all bits
+ self._inverted_ = self.__class__(~self._value_)
+ else:
+ # calculate flags not in this member
+ self._inverted_ = self.__class__(self._flag_mask_ ^ self._value_)
+ if isinstance(self._inverted_, self.__class__):
+ self._inverted_._inverted_ = self
+ return self._inverted_
+
+ __rand__ = __and__
+ __ror__ = __or__
+ __rxor__ = __xor__
-class IntFlag(int, Flag):
+class IntFlag(int, ReprEnum, Flag, boundary=KEEP):
"""
Support for integer-based Flags
"""
- @classmethod
- def _missing_(cls, value):
- """
- Returns member (possibly creating it) if one can be found for value.
- """
- if not isinstance(value, int):
- raise ValueError("%r is not a valid %s" % (value, cls.__qualname__))
- new_member = cls._create_pseudo_member_(value)
- return new_member
-
- @classmethod
- def _create_pseudo_member_(cls, value):
- """
- Create a composite member iff value contains only members.
- """
- pseudo_member = cls._value2member_map_.get(value, None)
- if pseudo_member is None:
- need_to_create = [value]
- # get unaccounted for bits
- _, extra_flags = _decompose(cls, value)
- # timer = 10
- while extra_flags:
- # timer -= 1
- bit = _high_bit(extra_flags)
- flag_value = 2 ** bit
- if (flag_value not in cls._value2member_map_ and
- flag_value not in need_to_create
- ):
- need_to_create.append(flag_value)
- if extra_flags == -flag_value:
- extra_flags = 0
- else:
- extra_flags ^= flag_value
- for value in reversed(need_to_create):
- # construct singleton pseudo-members
- pseudo_member = int.__new__(cls, value)
- pseudo_member._name_ = None
- pseudo_member._value_ = value
- # use setdefault in case another thread already created a composite
- # with this value
- pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member)
- return pseudo_member
-
- def __or__(self, other):
- if not isinstance(other, (self.__class__, int)):
- return NotImplemented
- result = self.__class__(self._value_ | self.__class__(other)._value_)
- return result
-
- def __and__(self, other):
- if not isinstance(other, (self.__class__, int)):
- return NotImplemented
- return self.__class__(self._value_ & self.__class__(other)._value_)
-
- def __xor__(self, other):
- if not isinstance(other, (self.__class__, int)):
- return NotImplemented
- return self.__class__(self._value_ ^ self.__class__(other)._value_)
-
- __ror__ = __or__
- __rand__ = __and__
- __rxor__ = __xor__
-
- def __invert__(self):
- result = self.__class__(~self._value_)
- return result
-
def _high_bit(value):
"""
@@ -1023,31 +1576,478 @@
(enumeration, alias_details))
return enumeration
-def _decompose(flag, value):
+def _power_of_two(value):
+ if value < 1:
+ return False
+ return value == 2 ** _high_bit(value)
+
+def global_enum_repr(self):
"""
- Extract all members from the value.
+ use module.enum_name instead of class.enum_name
+
+ the module is the last module in case of a multi-module name
"""
- # _decompose is only called if the value is not named
- not_covered = value
- negative = value < 0
- members = []
- for member in flag:
- member_value = member.value
- if member_value and member_value & value == member_value:
- members.append(member)
- not_covered &= ~member_value
- if not negative:
- tmp = not_covered
- while tmp:
- flag_value = 2 ** _high_bit(tmp)
- if flag_value in flag._value2member_map_:
- members.append(flag._value2member_map_[flag_value])
- not_covered &= ~flag_value
- tmp &= ~flag_value
- if not members and value in flag._value2member_map_:
- members.append(flag._value2member_map_[value])
- members.sort(key=lambda m: m._value_, reverse=True)
- if len(members) > 1 and members[0].value == value:
- # we have the breakdown, don't need the value member itself
- members.pop(0)
- return members, not_covered
+ module = self.__class__.__module__.split('.')[-1]
+ return '%s.%s' % (module, self._name_)
+
+def global_flag_repr(self):
+ """
+ use module.flag_name instead of class.flag_name
+
+ the module is the last module in case of a multi-module name
+ """
+ module = self.__class__.__module__.split('.')[-1]
+ cls_name = self.__class__.__name__
+ if self._name_ is None:
+ return "%s.%s(%r)" % (module, cls_name, self._value_)
+ if _is_single_bit(self):
+ return '%s.%s' % (module, self._name_)
+ if self._boundary_ is not FlagBoundary.KEEP:
+ return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')])
+ else:
+ name = []
+ for n in self._name_.split('|'):
+ if n[0].isdigit():
+ name.append(n)
+ else:
+ name.append('%s.%s' % (module, n))
+ return '|'.join(name)
+
+def global_str(self):
+ """
+ use enum_name instead of class.enum_name
+ """
+ if self._name_ is None:
+ cls_name = self.__class__.__name__
+ return "%s(%r)" % (cls_name, self._value_)
+ else:
+ return self._name_
+
+def global_enum(cls, update_str=False):
+ """
+ decorator that makes the repr() of an enum member reference its module
+ instead of its class; also exports all members to the enum's module's
+ global namespace
+ """
+ if issubclass(cls, Flag):
+ cls.__repr__ = global_flag_repr
+ else:
+ cls.__repr__ = global_enum_repr
+ if not issubclass(cls, ReprEnum) or update_str:
+ cls.__str__ = global_str
+ sys.modules[cls.__module__].__dict__.update(cls.__members__)
+ return cls
+
+def _simple_enum(etype=Enum, *, boundary=None, use_args=None):
+ """
+ Class decorator that converts a normal class into an :class:`Enum`. No
+ safety checks are done, and some advanced behavior (such as
+ :func:`__init_subclass__`) is not available. Enum creation can be faster
+ using :func:`simple_enum`.
+
+ >>> from enum import Enum, _simple_enum
+ >>> @_simple_enum(Enum)
+ ... class Color:
+ ... RED = auto()
+ ... GREEN = auto()
+ ... BLUE = auto()
+ >>> Color
+ <enum 'Color'>
+ """
+ def convert_class(cls):
+ nonlocal use_args
+ cls_name = cls.__name__
+ if use_args is None:
+ use_args = etype._use_args_
+ __new__ = cls.__dict__.get('__new__')
+ if __new__ is not None:
+ new_member = __new__.__func__
+ else:
+ new_member = etype._member_type_.__new__
+ attrs = {}
+ body = {}
+ if __new__ is not None:
+ body['__new_member__'] = new_member
+ body['_new_member_'] = new_member
+ body['_use_args_'] = use_args
+ body['_generate_next_value_'] = gnv = etype._generate_next_value_
+ body['_member_names_'] = member_names = []
+ body['_member_map_'] = member_map = {}
+ body['_value2member_map_'] = value2member_map = {}
+ body['_unhashable_values_'] = []
+ body['_member_type_'] = member_type = etype._member_type_
+ body['_value_repr_'] = etype._value_repr_
+ if issubclass(etype, Flag):
+ body['_boundary_'] = boundary or etype._boundary_
+ body['_flag_mask_'] = None
+ body['_all_bits_'] = None
+ body['_singles_mask_'] = None
+ body['_inverted_'] = None
+ body['__or__'] = Flag.__or__
+ body['__xor__'] = Flag.__xor__
+ body['__and__'] = Flag.__and__
+ body['__ror__'] = Flag.__ror__
+ body['__rxor__'] = Flag.__rxor__
+ body['__rand__'] = Flag.__rand__
+ body['__invert__'] = Flag.__invert__
+ for name, obj in cls.__dict__.items():
+ if name in ('__dict__', '__weakref__'):
+ continue
+ if _is_dunder(name) or _is_private(cls_name, name) or _is_sunder(name) or _is_descriptor(obj):
+ body[name] = obj
+ else:
+ attrs[name] = obj
+ if cls.__dict__.get('__doc__') is None:
+ body['__doc__'] = 'An enumeration.'
+ #
+ # double check that repr and friends are not the mixin's or various
+ # things break (such as pickle)
+ # however, if the method is defined in the Enum itself, don't replace
+ # it
+ enum_class = type(cls_name, (etype, ), body, boundary=boundary, _simple=True)
+ for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'):
+ if name not in body:
+ # check for mixin overrides before replacing
+ enum_method = getattr(etype, name)
+ found_method = getattr(enum_class, name)
+ object_method = getattr(object, name)
+ data_type_method = getattr(member_type, name)
+ if found_method in (data_type_method, object_method):
+ setattr(enum_class, name, enum_method)
+ gnv_last_values = []
+ if issubclass(enum_class, Flag):
+ # Flag / IntFlag
+ single_bits = multi_bits = 0
+ for name, value in attrs.items():
+ if isinstance(value, auto) and auto.value is _auto_null:
+ value = gnv(name, 1, len(member_names), gnv_last_values)
+ if value in value2member_map:
+ # an alias to an existing member
+ redirect = property()
+ redirect.__set_name__(enum_class, name)
+ setattr(enum_class, name, redirect)
+ member_map[name] = value2member_map[value]
+ else:
+ # create the member
+ if use_args:
+ if not isinstance(value, tuple):
+ value = (value, )
+ member = new_member(enum_class, *value)
+ value = value[0]
+ else:
+ member = new_member(enum_class)
+ if __new__ is None:
+ member._value_ = value
+ member._name_ = name
+ member.__objclass__ = enum_class
+ member.__init__(value)
+ redirect = property()
+ redirect.__set_name__(enum_class, name)
+ setattr(enum_class, name, redirect)
+ member_map[name] = member
+ member._sort_order_ = len(member_names)
+ value2member_map[value] = member
+ if _is_single_bit(value):
+ # not a multi-bit alias, record in _member_names_ and _flag_mask_
+ member_names.append(name)
+ single_bits |= value
+ else:
+ multi_bits |= value
+ gnv_last_values.append(value)
+ enum_class._flag_mask_ = single_bits | multi_bits
+ enum_class._singles_mask_ = single_bits
+ enum_class._all_bits_ = 2 ** ((single_bits|multi_bits).bit_length()) - 1
+ # set correct __iter__
+ member_list = [m._value_ for m in enum_class]
+ if member_list != sorted(member_list):
+ enum_class._iter_member_ = enum_class._iter_member_by_def_
+ else:
+ # Enum / IntEnum / StrEnum
+ for name, value in attrs.items():
+ if isinstance(value, auto):
+ if value.value is _auto_null:
+ value.value = gnv(name, 1, len(member_names), gnv_last_values)
+ value = value.value
+ if value in value2member_map:
+ # an alias to an existing member
+ redirect = property()
+ redirect.__set_name__(enum_class, name)
+ setattr(enum_class, name, redirect)
+ member_map[name] = value2member_map[value]
+ else:
+ # create the member
+ if use_args:
+ if not isinstance(value, tuple):
+ value = (value, )
+ member = new_member(enum_class, *value)
+ value = value[0]
+ else:
+ member = new_member(enum_class)
+ if __new__ is None:
+ member._value_ = value
+ member._name_ = name
+ member.__objclass__ = enum_class
+ member.__init__(value)
+ member._sort_order_ = len(member_names)
+ redirect = property()
+ redirect.__set_name__(enum_class, name)
+ setattr(enum_class, name, redirect)
+ member_map[name] = member
+ value2member_map[value] = member
+ member_names.append(name)
+ gnv_last_values.append(value)
+ if '__new__' in body:
+ enum_class.__new_member__ = enum_class.__new__
+ enum_class.__new__ = Enum.__new__
+ return enum_class
+ return convert_class
+
+@_simple_enum(StrEnum)
+class EnumCheck:
+ """
+ various conditions to check an enumeration for
+ """
+ CONTINUOUS = "no skipped integer values"
+ NAMED_FLAGS = "multi-flag aliases may not contain unnamed flags"
+ UNIQUE = "one name per value"
+CONTINUOUS, NAMED_FLAGS, UNIQUE = EnumCheck
+
+
+class verify:
+ """
+ Check an enumeration for various constraints. (see EnumCheck)
+ """
+ def __init__(self, *checks):
+ self.checks = checks
+ def __call__(self, enumeration):
+ checks = self.checks
+ cls_name = enumeration.__name__
+ if Flag is not None and issubclass(enumeration, Flag):
+ enum_type = 'flag'
+ elif issubclass(enumeration, Enum):
+ enum_type = 'enum'
+ else:
+ raise TypeError("the 'verify' decorator only works with Enum and Flag")
+ for check in checks:
+ if check is UNIQUE:
+ # check for duplicate names
+ duplicates = []
+ for name, member in enumeration.__members__.items():
+ if name != member.name:
+ duplicates.append((name, member.name))
+ if duplicates:
+ alias_details = ', '.join(
+ ["%s -> %s" % (alias, name) for (alias, name) in duplicates])
+ raise ValueError('aliases found in %r: %s' %
+ (enumeration, alias_details))
+ elif check is CONTINUOUS:
+ values = set(e.value for e in enumeration)
+ if len(values) < 2:
+ continue
+ low, high = min(values), max(values)
+ missing = []
+ if enum_type == 'flag':
+ # check for powers of two
+ for i in range(_high_bit(low)+1, _high_bit(high)):
+ if 2**i not in values:
+ missing.append(2**i)
+ elif enum_type == 'enum':
+ # check for powers of one
+ for i in range(low+1, high):
+ if i not in values:
+ missing.append(i)
+ else:
+ raise Exception('verify: unknown type %r' % enum_type)
+ if missing:
+ raise ValueError(('invalid %s %r: missing values %s' % (
+ enum_type, cls_name, ', '.join((str(m) for m in missing)))
+ )[:256])
+ # limit max length to protect against DOS attacks
+ elif check is NAMED_FLAGS:
+ # examine each alias and check for unnamed flags
+ member_names = enumeration._member_names_
+ member_values = [m.value for m in enumeration]
+ missing_names = []
+ missing_value = 0
+ for name, alias in enumeration._member_map_.items():
+ if name in member_names:
+ # not an alias
+ continue
+ if alias.value < 0:
+ # negative numbers are not checked
+ continue
+ values = list(_iter_bits_lsb(alias.value))
+ missed = [v for v in values if v not in member_values]
+ if missed:
+ missing_names.append(name)
+ missing_value |= reduce(_or_, missed)
+ if missing_names:
+ if len(missing_names) == 1:
+ alias = 'alias %s is missing' % missing_names[0]
+ else:
+ alias = 'aliases %s and %s are missing' % (
+ ', '.join(missing_names[:-1]), missing_names[-1]
+ )
+ if _is_single_bit(missing_value):
+ value = 'value 0x%x' % missing_value
+ else:
+ value = 'combined values of 0x%x' % missing_value
+ raise ValueError(
+ 'invalid Flag %r: %s %s [use enum.show_flag_values(value) for details]'
+ % (cls_name, alias, value)
+ )
+ return enumeration
+
+def _test_simple_enum(checked_enum, simple_enum):
+ """
+ A function that can be used to test an enum created with :func:`_simple_enum`
+ against the version created by subclassing :class:`Enum`::
+
+ >>> from enum import Enum, _simple_enum, _test_simple_enum
+ >>> @_simple_enum(Enum)
+ ... class Color:
+ ... RED = auto()
+ ... GREEN = auto()
+ ... BLUE = auto()
+ >>> class CheckedColor(Enum):
+ ... RED = auto()
+ ... GREEN = auto()
+ ... BLUE = auto()
+ >>> _test_simple_enum(CheckedColor, Color)
+
+ If differences are found, a :exc:`TypeError` is raised.
+ """
+ failed = []
+ if checked_enum.__dict__ != simple_enum.__dict__:
+ checked_dict = checked_enum.__dict__
+ checked_keys = list(checked_dict.keys())
+ simple_dict = simple_enum.__dict__
+ simple_keys = list(simple_dict.keys())
+ member_names = set(
+ list(checked_enum._member_map_.keys())
+ + list(simple_enum._member_map_.keys())
+ )
+ for key in set(checked_keys + simple_keys):
+ if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'):
+ # keys known to be different, or very long
+ continue
+ elif key in member_names:
+ # members are checked below
+ continue
+ elif key not in simple_keys:
+ failed.append("missing key: %r" % (key, ))
+ elif key not in checked_keys:
+ failed.append("extra key: %r" % (key, ))
+ else:
+ checked_value = checked_dict[key]
+ simple_value = simple_dict[key]
+ if callable(checked_value) or isinstance(checked_value, bltns.property):
+ continue
+ if key == '__doc__':
+ # remove all spaces/tabs
+ compressed_checked_value = checked_value.replace(' ','').replace('\t','')
+ compressed_simple_value = simple_value.replace(' ','').replace('\t','')
+ if compressed_checked_value != compressed_simple_value:
+ failed.append("%r:\n %s\n %s" % (
+ key,
+ "checked -> %r" % (checked_value, ),
+ "simple -> %r" % (simple_value, ),
+ ))
+ elif checked_value != simple_value:
+ failed.append("%r:\n %s\n %s" % (
+ key,
+ "checked -> %r" % (checked_value, ),
+ "simple -> %r" % (simple_value, ),
+ ))
+ failed.sort()
+ for name in member_names:
+ failed_member = []
+ if name not in simple_keys:
+ failed.append('missing member from simple enum: %r' % name)
+ elif name not in checked_keys:
+ failed.append('extra member in simple enum: %r' % name)
+ else:
+ checked_member_dict = checked_enum[name].__dict__
+ checked_member_keys = list(checked_member_dict.keys())
+ simple_member_dict = simple_enum[name].__dict__
+ simple_member_keys = list(simple_member_dict.keys())
+ for key in set(checked_member_keys + simple_member_keys):
+ if key in ('__module__', '__objclass__', '_inverted_'):
+ # keys known to be different or absent
+ continue
+ elif key not in simple_member_keys:
+ failed_member.append("missing key %r not in the simple enum member %r" % (key, name))
+ elif key not in checked_member_keys:
+ failed_member.append("extra key %r in simple enum member %r" % (key, name))
+ else:
+ checked_value = checked_member_dict[key]
+ simple_value = simple_member_dict[key]
+ if checked_value != simple_value:
+ failed_member.append("%r:\n %s\n %s" % (
+ key,
+ "checked member -> %r" % (checked_value, ),
+ "simple member -> %r" % (simple_value, ),
+ ))
+ if failed_member:
+ failed.append('%r member mismatch:\n %s' % (
+ name, '\n '.join(failed_member),
+ ))
+ for method in (
+ '__str__', '__repr__', '__reduce_ex__', '__format__',
+ '__getnewargs_ex__', '__getnewargs__', '__reduce_ex__', '__reduce__'
+ ):
+ if method in simple_keys and method in checked_keys:
+ # cannot compare functions, and it exists in both, so we're good
+ continue
+ elif method not in simple_keys and method not in checked_keys:
+ # method is inherited -- check it out
+ checked_method = getattr(checked_enum, method, None)
+ simple_method = getattr(simple_enum, method, None)
+ if hasattr(checked_method, '__func__'):
+ checked_method = checked_method.__func__
+ simple_method = simple_method.__func__
+ if checked_method != simple_method:
+ failed.append("%r: %-30s %s" % (
+ method,
+ "checked -> %r" % (checked_method, ),
+ "simple -> %r" % (simple_method, ),
+ ))
+ else:
+ # if the method existed in only one of the enums, it will have been caught
+ # in the first checks above
+ pass
+ if failed:
+ raise TypeError('enum mismatch:\n %s' % '\n '.join(failed))
+
+def _old_convert_(etype, name, module, filter, source=None, *, boundary=None):
+ """
+ Create a new Enum subclass that replaces a collection of global constants
+ """
+ # convert all constants from source (or module) that pass filter() to
+ # a new Enum called name, and export the enum and its members back to
+ # module;
+ # also, replace the __reduce_ex__ method so unpickling works in
+ # previous Python versions
+ module_globals = sys.modules[module].__dict__
+ if source:
+ source = source.__dict__
+ else:
+ source = module_globals
+ # _value2member_map_ is populated in the same order every time
+ # for a consistent reverse mapping of number to name when there
+ # are multiple names for the same number.
+ members = [
+ (name, value)
+ for name, value in source.items()
+ if filter(name)]
+ try:
+ # sort by value
+ members.sort(key=lambda t: (t[1], t[0]))
+ except TypeError:
+ # unless some values aren't comparable, in which case sort by name
+ members.sort(key=lambda t: t[0])
+ cls = etype(name, members, module=module, boundary=boundary or KEEP)
+ cls.__reduce_ex__ = _reduce_ex_by_global_name
+ return cls
+
+_stdlib_enums = IntEnum, StrEnum, IntFlag
diff --git a/common/py3-stdlib/fileinput.py b/common/py3-stdlib/fileinput.py
index 3bd1990..1b25f28 100644
--- a/common/py3-stdlib/fileinput.py
+++ b/common/py3-stdlib/fileinput.py
@@ -217,15 +217,10 @@
EncodingWarning, 2)
# restrict mode argument to reading modes
- if mode not in ('r', 'rU', 'U', 'rb'):
- raise ValueError("FileInput opening mode must be one of "
- "'r', 'rU', 'U' and 'rb'")
- if 'U' in mode:
- import warnings
- warnings.warn("'U' mode is deprecated",
- DeprecationWarning, 2)
+ if mode not in ('r', 'rb'):
+ raise ValueError("FileInput opening mode must be 'r' or 'rb'")
self._mode = mode
- self._write_mode = mode.replace('r', 'w') if 'U' not in mode else 'w'
+ self._write_mode = mode.replace('r', 'w')
if openhook:
if inplace:
raise ValueError("FileInput cannot use an opening hook in inplace mode")
@@ -262,21 +257,6 @@
self.nextfile()
# repeat with next file
- def __getitem__(self, i):
- import warnings
- warnings.warn(
- "Support for indexing FileInput objects is deprecated. "
- "Use iterator protocol instead.",
- DeprecationWarning,
- stacklevel=2
- )
- if i != self.lineno():
- raise RuntimeError("accessing lines out of order")
- try:
- return self.__next__()
- except StopIteration:
- raise IndexError("end of input reached")
-
def nextfile(self):
savestdout = self._savestdout
self._savestdout = None
diff --git a/common/py3-stdlib/fnmatch.py b/common/py3-stdlib/fnmatch.py
index fee59bf..d5e296f 100644
--- a/common/py3-stdlib/fnmatch.py
+++ b/common/py3-stdlib/fnmatch.py
@@ -16,12 +16,6 @@
__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"]
-# Build a thread-safe incrementing counter to help create unique regexp group
-# names across calls.
-from itertools import count
-_nextgroupnum = count().__next__
-del count
-
def fnmatch(name, pat):
"""Test whether FILENAME matches PATTERN.
@@ -41,7 +35,7 @@
pat = os.path.normcase(pat)
return fnmatchcase(name, pat)
-@functools.lru_cache(maxsize=256, typed=True)
+@functools.lru_cache(maxsize=32768, typed=True)
def _compile_pattern(pat):
if isinstance(pat, bytes):
pat_str = str(pat, 'ISO-8859-1')
@@ -165,17 +159,10 @@
# Now deal with STAR fixed STAR fixed ...
# For an interior `STAR fixed` pairing, we want to do a minimal
# .*? match followed by `fixed`, with no possibility of backtracking.
- # We can't spell that directly, but can trick it into working by matching
- # .*?fixed
- # in a lookahead assertion, save the matched part in a group, then
- # consume that group via a backreference. If the overall match fails,
- # the lookahead assertion won't try alternatives. So the translation is:
- # (?=(?P<name>.*?fixed))(?P=name)
- # Group names are created as needed: g0, g1, g2, ...
- # The numbers are obtained from _nextgroupnum() to ensure they're unique
- # across calls and across threads. This is because people rely on the
- # undocumented ability to join multiple translate() results together via
- # "|" to build large regexps matching "one of many" shell patterns.
+ # Atomic groups ("(?>...)") allow us to spell that directly.
+ # Note: people rely on the undocumented ability to join multiple
+ # translate() results together via "|" to build large regexps matching
+ # "one of many" shell patterns.
while i < n:
assert inp[i] is STAR
i += 1
@@ -192,8 +179,7 @@
add(".*")
add(fixed)
else:
- groupnum = _nextgroupnum()
- add(f"(?=(?P<g{groupnum}>.*?{fixed}))(?P=g{groupnum})")
+ add(f"(?>.*?{fixed})")
assert i == n
res = "".join(res)
return fr'(?s:{res})\Z'
diff --git a/common/py3-stdlib/fractions.py b/common/py3-stdlib/fractions.py
index 96047be..f9ac882 100644
--- a/common/py3-stdlib/fractions.py
+++ b/common/py3-stdlib/fractions.py
@@ -1,7 +1,7 @@
# Originally contributed by Sjoerd Mullender.
# Significantly modified by Jeffrey Yasskin <jyasskin at gmail.com>.
-"""Fraction, infinite-precision, real numbers."""
+"""Fraction, infinite-precision, rational numbers."""
from decimal import Decimal
import math
@@ -21,17 +21,17 @@
_PyHASH_INF = sys.hash_info.inf
_RATIONAL_FORMAT = re.compile(r"""
- \A\s* # optional whitespace at the start, then
- (?P<sign>[-+]?) # an optional sign, then
- (?=\d|\.\d) # lookahead for digit or .digit
- (?P<num>\d*) # numerator (possibly empty)
- (?: # followed by
- (?:/(?P<denom>\d+))? # an optional denominator
- | # or
- (?:\.(?P<decimal>\d*))? # an optional fractional part
- (?:E(?P<exp>[-+]?\d+))? # and optional exponent
+ \A\s* # optional whitespace at the start,
+ (?P<sign>[-+]?) # an optional sign, then
+ (?=\d|\.\d) # lookahead for digit or .digit
+ (?P<num>\d*|\d+(_\d+)*) # numerator (possibly empty)
+ (?: # followed by
+ (?:/(?P<denom>\d+(_\d+)*))? # an optional denominator
+ | # or
+ (?:\.(?P<decimal>d*|\d+(_\d+)*))? # an optional fractional part
+ (?:E(?P<exp>[-+]?\d+(_\d+)*))? # and optional exponent
)
- \s*\Z # and optional whitespace to finish
+ \s*\Z # and optional whitespace to finish
""", re.VERBOSE | re.IGNORECASE)
@@ -122,6 +122,7 @@
denominator = 1
decimal = m.group('decimal')
if decimal:
+ decimal = decimal.replace('_', '')
scale = 10**len(decimal)
numerator = numerator * scale + int(decimal)
denominator *= scale
@@ -593,8 +594,15 @@
"""abs(a)"""
return Fraction(abs(a._numerator), a._denominator, _normalize=False)
+ def __int__(a, _index=operator.index):
+ """int(a)"""
+ if a._numerator < 0:
+ return _index(-(-a._numerator // a._denominator))
+ else:
+ return _index(a._numerator // a._denominator)
+
def __trunc__(a):
- """trunc(a)"""
+ """math.trunc(a)"""
if a._numerator < 0:
return -(-a._numerator // a._denominator)
else:
@@ -735,7 +743,7 @@
# support for pickling, copy, and deepcopy
def __reduce__(self):
- return (self.__class__, (str(self),))
+ return (self.__class__, (self._numerator, self._denominator))
def __copy__(self):
if type(self) == Fraction:
diff --git a/common/py3-stdlib/functools.py b/common/py3-stdlib/functools.py
index 305ceb4..43ead51 100644
--- a/common/py3-stdlib/functools.py
+++ b/common/py3-stdlib/functools.py
@@ -86,84 +86,84 @@
# infinite recursion that could occur when the operator dispatch logic
# detects a NotImplemented result and then calls a reflected method.
-def _gt_from_lt(self, other, NotImplemented=NotImplemented):
+def _gt_from_lt(self, other):
'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).'
op_result = type(self).__lt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result and self != other
-def _le_from_lt(self, other, NotImplemented=NotImplemented):
+def _le_from_lt(self, other):
'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).'
op_result = type(self).__lt__(self, other)
if op_result is NotImplemented:
return op_result
return op_result or self == other
-def _ge_from_lt(self, other, NotImplemented=NotImplemented):
+def _ge_from_lt(self, other):
'Return a >= b. Computed by @total_ordering from (not a < b).'
op_result = type(self).__lt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result
-def _ge_from_le(self, other, NotImplemented=NotImplemented):
+def _ge_from_le(self, other):
'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).'
op_result = type(self).__le__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result or self == other
-def _lt_from_le(self, other, NotImplemented=NotImplemented):
+def _lt_from_le(self, other):
'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).'
op_result = type(self).__le__(self, other)
if op_result is NotImplemented:
return op_result
return op_result and self != other
-def _gt_from_le(self, other, NotImplemented=NotImplemented):
+def _gt_from_le(self, other):
'Return a > b. Computed by @total_ordering from (not a <= b).'
op_result = type(self).__le__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result
-def _lt_from_gt(self, other, NotImplemented=NotImplemented):
+def _lt_from_gt(self, other):
'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).'
op_result = type(self).__gt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result and self != other
-def _ge_from_gt(self, other, NotImplemented=NotImplemented):
+def _ge_from_gt(self, other):
'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).'
op_result = type(self).__gt__(self, other)
if op_result is NotImplemented:
return op_result
return op_result or self == other
-def _le_from_gt(self, other, NotImplemented=NotImplemented):
+def _le_from_gt(self, other):
'Return a <= b. Computed by @total_ordering from (not a > b).'
op_result = type(self).__gt__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result
-def _le_from_ge(self, other, NotImplemented=NotImplemented):
+def _le_from_ge(self, other):
'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).'
op_result = type(self).__ge__(self, other)
if op_result is NotImplemented:
return op_result
return not op_result or self == other
-def _gt_from_ge(self, other, NotImplemented=NotImplemented):
+def _gt_from_ge(self, other):
'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).'
op_result = type(self).__ge__(self, other)
if op_result is NotImplemented:
return op_result
return op_result and self != other
-def _lt_from_ge(self, other, NotImplemented=NotImplemented):
+def _lt_from_ge(self, other):
'Return a < b. Computed by @total_ordering from (not a >= b).'
op_result = type(self).__ge__(self, other)
if op_result is NotImplemented:
@@ -838,8 +838,16 @@
dispatch_cache[cls] = impl
return impl
+ def _is_union_type(cls):
+ from typing import get_origin, Union
+ return get_origin(cls) in {Union, types.UnionType}
+
def _is_valid_dispatch_type(cls):
- return isinstance(cls, type) and not isinstance(cls, GenericAlias)
+ if isinstance(cls, type):
+ return True
+ from typing import get_args
+ return (_is_union_type(cls) and
+ all(isinstance(arg, type) for arg in get_args(cls)))
def register(cls, func=None):
"""generic_func.register(cls, func) -> func
@@ -855,7 +863,7 @@
if func is not None:
raise TypeError(
f"Invalid first argument to `register()`. "
- f"{cls!r} is not a class."
+ f"{cls!r} is not a class or union type."
)
ann = getattr(cls, '__annotations__', {})
if not ann:
@@ -870,12 +878,24 @@
from typing import get_type_hints
argname, cls = next(iter(get_type_hints(func).items()))
if not _is_valid_dispatch_type(cls):
- raise TypeError(
- f"Invalid annotation for {argname!r}. "
- f"{cls!r} is not a class."
- )
+ if _is_union_type(cls):
+ raise TypeError(
+ f"Invalid annotation for {argname!r}. "
+ f"{cls!r} not all arguments are classes."
+ )
+ else:
+ raise TypeError(
+ f"Invalid annotation for {argname!r}. "
+ f"{cls!r} is not a class."
+ )
- registry[cls] = func
+ if _is_union_type(cls):
+ from typing import get_args
+
+ for arg in get_args(cls):
+ registry[arg] = func
+ else:
+ registry[cls] = func
if cache_token is None and hasattr(cls, '__abstractmethods__'):
cache_token = get_cache_token()
dispatch_cache.clear()
diff --git a/common/py3-stdlib/gettext.py b/common/py3-stdlib/gettext.py
index 77b67ae..6c5ec4e 100644
--- a/common/py3-stdlib/gettext.py
+++ b/common/py3-stdlib/gettext.py
@@ -52,11 +52,10 @@
__all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
- 'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
- 'bind_textdomain_codeset',
- 'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
- 'ldngettext', 'lngettext', 'ngettext',
- 'pgettext', 'dpgettext', 'npgettext', 'dnpgettext',
+ 'bindtextdomain', 'find', 'translation', 'install',
+ 'textdomain', 'dgettext', 'dngettext', 'gettext',
+ 'ngettext', 'pgettext', 'dpgettext', 'npgettext',
+ 'dnpgettext'
]
_default_localedir = os.path.join(sys.base_prefix, 'share', 'locale')
@@ -83,6 +82,7 @@
(?P<INVALID>\w+|.) # invalid token
""", re.VERBOSE|re.DOTALL)
+
def _tokenize(plural):
for mo in re.finditer(_token_pattern, plural):
kind = mo.lastgroup
@@ -94,12 +94,14 @@
yield value
yield ''
+
def _error(value):
if value:
return ValueError('unexpected token in plural form: %s' % value)
else:
return ValueError('unexpected end of plural form')
+
_binary_ops = (
('||',),
('&&',),
@@ -111,6 +113,7 @@
_binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops}
_c2py_ops = {'||': 'or', '&&': 'and', '/': '//'}
+
def _parse(tokens, priority=-1):
result = ''
nexttok = next(tokens)
@@ -160,6 +163,7 @@
return result, nexttok
+
def _as_int(n):
try:
i = round(n)
@@ -172,6 +176,7 @@
DeprecationWarning, 4)
return n
+
def c2py(plural):
"""Gets a C expression as used in PO files for plural forms and returns a
Python function that implements an equivalent expression.
@@ -250,12 +255,10 @@
return ret
-
class NullTranslations:
def __init__(self, fp=None):
self._info = {}
self._charset = None
- self._output_charset = None
self._fallback = None
if fp is not None:
self._parse(fp)
@@ -274,20 +277,6 @@
return self._fallback.gettext(message)
return message
- def lgettext(self, message):
- import warnings
- warnings.warn('lgettext() is deprecated, use gettext() instead',
- DeprecationWarning, 2)
- import locale
- if self._fallback:
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\blgettext\b.*',
- DeprecationWarning)
- return self._fallback.lgettext(message)
- if self._output_charset:
- return message.encode(self._output_charset)
- return message.encode(locale.getpreferredencoding())
-
def ngettext(self, msgid1, msgid2, n):
if self._fallback:
return self._fallback.ngettext(msgid1, msgid2, n)
@@ -296,24 +285,6 @@
else:
return msgid2
- def lngettext(self, msgid1, msgid2, n):
- import warnings
- warnings.warn('lngettext() is deprecated, use ngettext() instead',
- DeprecationWarning, 2)
- import locale
- if self._fallback:
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\blngettext\b.*',
- DeprecationWarning)
- return self._fallback.lngettext(msgid1, msgid2, n)
- if n == 1:
- tmsg = msgid1
- else:
- tmsg = msgid2
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- return tmsg.encode(locale.getpreferredencoding())
-
def pgettext(self, context, message):
if self._fallback:
return self._fallback.pgettext(context, message)
@@ -333,24 +304,11 @@
def charset(self):
return self._charset
- def output_charset(self):
- import warnings
- warnings.warn('output_charset() is deprecated',
- DeprecationWarning, 2)
- return self._output_charset
-
- def set_output_charset(self, charset):
- import warnings
- warnings.warn('set_output_charset() is deprecated',
- DeprecationWarning, 2)
- self._output_charset = charset
-
def install(self, names=None):
import builtins
builtins.__dict__['_'] = self.gettext
if names is not None:
- allowed = {'gettext', 'lgettext', 'lngettext',
- 'ngettext', 'npgettext', 'pgettext'}
+ allowed = {'gettext', 'ngettext', 'npgettext', 'pgettext'}
for name in allowed & set(names):
builtins.__dict__[name] = getattr(self, name)
@@ -460,39 +418,6 @@
masteridx += 8
transidx += 8
- def lgettext(self, message):
- import warnings
- warnings.warn('lgettext() is deprecated, use gettext() instead',
- DeprecationWarning, 2)
- import locale
- missing = object()
- tmsg = self._catalog.get(message, missing)
- if tmsg is missing:
- if self._fallback:
- return self._fallback.lgettext(message)
- tmsg = message
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- return tmsg.encode(locale.getpreferredencoding())
-
- def lngettext(self, msgid1, msgid2, n):
- import warnings
- warnings.warn('lngettext() is deprecated, use ngettext() instead',
- DeprecationWarning, 2)
- import locale
- try:
- tmsg = self._catalog[(msgid1, self.plural(n))]
- except KeyError:
- if self._fallback:
- return self._fallback.lngettext(msgid1, msgid2, n)
- if n == 1:
- tmsg = msgid1
- else:
- tmsg = msgid2
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- return tmsg.encode(locale.getpreferredencoding())
-
def gettext(self, message):
missing = object()
tmsg = self._catalog.get(message, missing)
@@ -575,13 +500,12 @@
return result
-
# a mapping between absolute .mo file path and Translation object
_translations = {}
-_unspecified = ['unspecified']
+
def translation(domain, localedir=None, languages=None,
- class_=None, fallback=False, codeset=_unspecified):
+ class_=None, fallback=False):
if class_ is None:
class_ = GNUTranslations
mofiles = find(domain, localedir, languages, all=True)
@@ -607,15 +531,6 @@
# are not used.
import copy
t = copy.copy(t)
- if codeset is not _unspecified:
- import warnings
- warnings.warn('parameter codeset is deprecated',
- DeprecationWarning, 2)
- if codeset:
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\bset_output_charset\b.*',
- DeprecationWarning)
- t.set_output_charset(codeset)
if result is None:
result = t
else:
@@ -623,16 +538,13 @@
return result
-def install(domain, localedir=None, codeset=_unspecified, names=None):
- t = translation(domain, localedir, fallback=True, codeset=codeset)
+def install(domain, localedir=None, *, names=None):
+ t = translation(domain, localedir, fallback=True)
t.install(names)
-
# a mapping b/w domains and locale directories
_localedirs = {}
-# a mapping b/w domains and codesets
-_localecodesets = {}
# current global domain, `messages' used for compatibility w/ GNU gettext
_current_domain = 'messages'
@@ -651,16 +563,6 @@
return _localedirs.get(domain, _default_localedir)
-def bind_textdomain_codeset(domain, codeset=None):
- import warnings
- warnings.warn('bind_textdomain_codeset() is deprecated',
- DeprecationWarning, 2)
- global _localecodesets
- if codeset is not None:
- _localecodesets[domain] = codeset
- return _localecodesets.get(domain)
-
-
def dgettext(domain, message):
try:
t = translation(domain, _localedirs.get(domain, None))
@@ -668,23 +570,6 @@
return message
return t.gettext(message)
-def ldgettext(domain, message):
- import warnings
- warnings.warn('ldgettext() is deprecated, use dgettext() instead',
- DeprecationWarning, 2)
- import locale
- codeset = _localecodesets.get(domain)
- try:
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\bparameter codeset\b.*',
- DeprecationWarning)
- t = translation(domain, _localedirs.get(domain, None), codeset=codeset)
- except OSError:
- return message.encode(codeset or locale.getpreferredencoding())
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\blgettext\b.*',
- DeprecationWarning)
- return t.lgettext(message)
def dngettext(domain, msgid1, msgid2, n):
try:
@@ -696,28 +581,6 @@
return msgid2
return t.ngettext(msgid1, msgid2, n)
-def ldngettext(domain, msgid1, msgid2, n):
- import warnings
- warnings.warn('ldngettext() is deprecated, use dngettext() instead',
- DeprecationWarning, 2)
- import locale
- codeset = _localecodesets.get(domain)
- try:
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\bparameter codeset\b.*',
- DeprecationWarning)
- t = translation(domain, _localedirs.get(domain, None), codeset=codeset)
- except OSError:
- if n == 1:
- tmsg = msgid1
- else:
- tmsg = msgid2
- return tmsg.encode(codeset or locale.getpreferredencoding())
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\blngettext\b.*',
- DeprecationWarning)
- return t.lngettext(msgid1, msgid2, n)
-
def dpgettext(domain, context, message):
try:
@@ -741,27 +604,10 @@
def gettext(message):
return dgettext(_current_domain, message)
-def lgettext(message):
- import warnings
- warnings.warn('lgettext() is deprecated, use gettext() instead',
- DeprecationWarning, 2)
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\bldgettext\b.*',
- DeprecationWarning)
- return ldgettext(_current_domain, message)
def ngettext(msgid1, msgid2, n):
return dngettext(_current_domain, msgid1, msgid2, n)
-def lngettext(msgid1, msgid2, n):
- import warnings
- warnings.warn('lngettext() is deprecated, use ngettext() instead',
- DeprecationWarning, 2)
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', r'.*\bldngettext\b.*',
- DeprecationWarning)
- return ldngettext(_current_domain, msgid1, msgid2, n)
-
def pgettext(context, message):
return dpgettext(_current_domain, context, message)
diff --git a/common/py3-stdlib/glob.py b/common/py3-stdlib/glob.py
index 9fc08f4..a725642 100644
--- a/common/py3-stdlib/glob.py
+++ b/common/py3-stdlib/glob.py
@@ -10,20 +10,26 @@
__all__ = ["glob", "iglob", "escape"]
-def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
+def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
+ include_hidden=False):
"""Return a list of paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
- fnmatch. However, unlike fnmatch, filenames starting with a
+ fnmatch. Unlike fnmatch, filenames starting with a
dot are special cases that are not matched by '*' and '?'
- patterns.
+ patterns by default.
- If recursive is true, the pattern '**' will match any files and
+ If `include_hidden` is true, the patterns '*', '?', '**' will match hidden
+ directories.
+
+ If `recursive` is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
"""
- return list(iglob(pathname, root_dir=root_dir, dir_fd=dir_fd, recursive=recursive))
+ return list(iglob(pathname, root_dir=root_dir, dir_fd=dir_fd, recursive=recursive,
+ include_hidden=include_hidden))
-def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False):
+def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
+ include_hidden=False):
"""Return an iterator which yields the paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
@@ -40,7 +46,8 @@
root_dir = os.fspath(root_dir)
else:
root_dir = pathname[:0]
- it = _iglob(pathname, root_dir, dir_fd, recursive, False)
+ it = _iglob(pathname, root_dir, dir_fd, recursive, False,
+ include_hidden=include_hidden)
if not pathname or recursive and _isrecursive(pathname[:2]):
try:
s = next(it) # skip empty string
@@ -50,7 +57,8 @@
pass
return it
-def _iglob(pathname, root_dir, dir_fd, recursive, dironly):
+def _iglob(pathname, root_dir, dir_fd, recursive, dironly,
+ include_hidden=False):
dirname, basename = os.path.split(pathname)
if not has_magic(pathname):
assert not dironly
@@ -64,15 +72,18 @@
return
if not dirname:
if recursive and _isrecursive(basename):
- yield from _glob2(root_dir, basename, dir_fd, dironly)
+ yield from _glob2(root_dir, basename, dir_fd, dironly,
+ include_hidden=include_hidden)
else:
- yield from _glob1(root_dir, basename, dir_fd, dironly)
+ yield from _glob1(root_dir, basename, dir_fd, dironly,
+ include_hidden=include_hidden)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
# contains magic characters (i.e. r'\\?\C:').
if dirname != pathname and has_magic(dirname):
- dirs = _iglob(dirname, root_dir, dir_fd, recursive, True)
+ dirs = _iglob(dirname, root_dir, dir_fd, recursive, True,
+ include_hidden=include_hidden)
else:
dirs = [dirname]
if has_magic(basename):
@@ -83,20 +94,21 @@
else:
glob_in_dir = _glob0
for dirname in dirs:
- for name in glob_in_dir(_join(root_dir, dirname), basename, dir_fd, dironly):
+ for name in glob_in_dir(_join(root_dir, dirname), basename, dir_fd, dironly,
+ include_hidden=include_hidden):
yield os.path.join(dirname, name)
# These 2 helper functions non-recursively glob inside a literal directory.
# They return a list of basenames. _glob1 accepts a pattern while _glob0
# takes a literal basename (so it only has to check for its existence).
-def _glob1(dirname, pattern, dir_fd, dironly):
+def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False):
names = _listdir(dirname, dir_fd, dironly)
- if not _ishidden(pattern):
- names = (x for x in names if not _ishidden(x))
+ if include_hidden or not _ishidden(pattern):
+ names = (x for x in names if include_hidden or not _ishidden(x))
return fnmatch.filter(names, pattern)
-def _glob0(dirname, basename, dir_fd, dironly):
+def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False):
if basename:
if _lexists(_join(dirname, basename), dir_fd):
return [basename]
@@ -118,10 +130,11 @@
# This helper function recursively yields relative pathnames inside a literal
# directory.
-def _glob2(dirname, pattern, dir_fd, dironly):
+def _glob2(dirname, pattern, dir_fd, dironly, include_hidden=False):
assert _isrecursive(pattern)
yield pattern[:0]
- yield from _rlistdir(dirname, dir_fd, dironly)
+ yield from _rlistdir(dirname, dir_fd, dironly,
+ include_hidden=include_hidden)
# If dironly is false, yields all file names inside a directory.
# If dironly is true, yields only directory names.
@@ -164,13 +177,14 @@
return list(it)
# Recursively yields relative pathnames inside a literal directory.
-def _rlistdir(dirname, dir_fd, dironly):
+def _rlistdir(dirname, dir_fd, dironly, include_hidden=False):
names = _listdir(dirname, dir_fd, dironly)
for x in names:
- if not _ishidden(x):
+ if include_hidden or not _ishidden(x):
yield x
path = _join(dirname, x) if dirname else x
- for y in _rlistdir(path, dir_fd, dironly):
+ for y in _rlistdir(path, dir_fd, dironly,
+ include_hidden=include_hidden):
yield _join(x, y)
diff --git a/common/py3-stdlib/graphlib.py b/common/py3-stdlib/graphlib.py
index 1c5d9a4..6365456 100644
--- a/common/py3-stdlib/graphlib.py
+++ b/common/py3-stdlib/graphlib.py
@@ -1,3 +1,5 @@
+from types import GenericAlias
+
__all__ = ["TopologicalSorter", "CycleError"]
_NODE_OUT = -1
@@ -244,3 +246,5 @@
node_group = self.get_ready()
yield from node_group
self.done(*node_group)
+
+ __class_getitem__ = classmethod(GenericAlias)
diff --git a/common/py3-stdlib/gzip.py b/common/py3-stdlib/gzip.py
index 475ec32..5b20e5b 100644
--- a/common/py3-stdlib/gzip.py
+++ b/common/py3-stdlib/gzip.py
@@ -399,6 +399,59 @@
return self._buffer.readline(size)
+def _read_exact(fp, n):
+ '''Read exactly *n* bytes from `fp`
+
+ This method is required because fp may be unbuffered,
+ i.e. return short reads.
+ '''
+ data = fp.read(n)
+ while len(data) < n:
+ b = fp.read(n - len(data))
+ if not b:
+ raise EOFError("Compressed file ended before the "
+ "end-of-stream marker was reached")
+ data += b
+ return data
+
+
+def _read_gzip_header(fp):
+ '''Read a gzip header from `fp` and progress to the end of the header.
+
+ Returns last mtime if header was present or None otherwise.
+ '''
+ magic = fp.read(2)
+ if magic == b'':
+ return None
+
+ if magic != b'\037\213':
+ raise BadGzipFile('Not a gzipped file (%r)' % magic)
+
+ (method, flag, last_mtime) = struct.unpack("<BBIxx", _read_exact(fp, 8))
+ if method != 8:
+ raise BadGzipFile('Unknown compression method')
+
+ if flag & FEXTRA:
+ # Read & discard the extra field, if present
+ extra_len, = struct.unpack("<H", _read_exact(fp, 2))
+ _read_exact(fp, extra_len)
+ if flag & FNAME:
+ # Read and discard a null-terminated string containing the filename
+ while True:
+ s = fp.read(1)
+ if not s or s==b'\000':
+ break
+ if flag & FCOMMENT:
+ # Read and discard a null-terminated string containing a comment
+ while True:
+ s = fp.read(1)
+ if not s or s==b'\000':
+ break
+ if flag & FHCRC:
+ _read_exact(fp, 2) # Read & discard the 16-bit header CRC
+ return last_mtime
+
+
class _GzipReader(_compression.DecompressReader):
def __init__(self, fp):
super().__init__(_PaddedFile(fp), zlib.decompressobj,
@@ -411,53 +464,11 @@
self._crc = zlib.crc32(b"")
self._stream_size = 0 # Decompressed size of unconcatenated stream
- def _read_exact(self, n):
- '''Read exactly *n* bytes from `self._fp`
-
- This method is required because self._fp may be unbuffered,
- i.e. return short reads.
- '''
-
- data = self._fp.read(n)
- while len(data) < n:
- b = self._fp.read(n - len(data))
- if not b:
- raise EOFError("Compressed file ended before the "
- "end-of-stream marker was reached")
- data += b
- return data
-
def _read_gzip_header(self):
- magic = self._fp.read(2)
- if magic == b'':
+ last_mtime = _read_gzip_header(self._fp)
+ if last_mtime is None:
return False
-
- if magic != b'\037\213':
- raise BadGzipFile('Not a gzipped file (%r)' % magic)
-
- (method, flag,
- self._last_mtime) = struct.unpack("<BBIxx", self._read_exact(8))
- if method != 8:
- raise BadGzipFile('Unknown compression method')
-
- if flag & FEXTRA:
- # Read & discard the extra field, if present
- extra_len, = struct.unpack("<H", self._read_exact(2))
- self._read_exact(extra_len)
- if flag & FNAME:
- # Read and discard a null-terminated string containing the filename
- while True:
- s = self._fp.read(1)
- if not s or s==b'\000':
- break
- if flag & FCOMMENT:
- # Read and discard a null-terminated string containing a comment
- while True:
- s = self._fp.read(1)
- if not s or s==b'\000':
- break
- if flag & FHCRC:
- self._read_exact(2) # Read & discard the 16-bit header CRC
+ self._last_mtime = last_mtime
return True
def read(self, size=-1):
@@ -520,7 +531,7 @@
# We check that the computed CRC and size of the
# uncompressed data matches the stored values. Note that the size
# stored is the true file size mod 2**32.
- crc32, isize = struct.unpack("<II", self._read_exact(8))
+ crc32, isize = struct.unpack("<II", _read_exact(self._fp, 8))
if crc32 != self._crc:
raise BadGzipFile("CRC check failed %s != %s" % (hex(crc32),
hex(self._crc)))
@@ -540,21 +551,69 @@
super()._rewind()
self._new_member = True
+
+def _create_simple_gzip_header(compresslevel: int,
+ mtime = None) -> bytes:
+ """
+ Write a simple gzip header with no extra fields.
+ :param compresslevel: Compresslevel used to determine the xfl bytes.
+ :param mtime: The mtime (must support conversion to a 32-bit integer).
+ :return: A bytes object representing the gzip header.
+ """
+ if mtime is None:
+ mtime = time.time()
+ if compresslevel == _COMPRESS_LEVEL_BEST:
+ xfl = 2
+ elif compresslevel == _COMPRESS_LEVEL_FAST:
+ xfl = 4
+ else:
+ xfl = 0
+ # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra
+ # fields added to header), mtime, xfl and os (255 for unknown OS).
+ return struct.pack("<BBBBLBB", 0x1f, 0x8b, 8, 0, int(mtime), xfl, 255)
+
+
def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=None):
"""Compress data in one shot and return the compressed string.
- Optional argument is the compression level, in range of 0-9.
+
+ compresslevel sets the compression level in range of 0-9.
+ mtime can be used to set the modification time. The modification time is
+ set to the current time by default.
"""
- buf = io.BytesIO()
- with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel, mtime=mtime) as f:
- f.write(data)
- return buf.getvalue()
+ if mtime == 0:
+ # Use zlib as it creates the header with 0 mtime by default.
+ # This is faster and with less overhead.
+ return zlib.compress(data, level=compresslevel, wbits=31)
+ header = _create_simple_gzip_header(compresslevel, mtime)
+ trailer = struct.pack("<LL", zlib.crc32(data), (len(data) & 0xffffffff))
+ # Wbits=-15 creates a raw deflate block.
+ return (header + zlib.compress(data, level=compresslevel, wbits=-15) +
+ trailer)
+
def decompress(data):
"""Decompress a gzip compressed string in one shot.
Return the decompressed string.
"""
- with GzipFile(fileobj=io.BytesIO(data)) as f:
- return f.read()
+ decompressed_members = []
+ while True:
+ fp = io.BytesIO(data)
+ if _read_gzip_header(fp) is None:
+ return b"".join(decompressed_members)
+ # Use a zlib raw deflate compressor
+ do = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
+ # Read all the data except the header
+ decompressed = do.decompress(data[fp.tell():])
+ if not do.eof or len(do.unused_data) < 8:
+ raise EOFError("Compressed file ended before the end-of-stream "
+ "marker was reached")
+ crc, length = struct.unpack("<II", do.unused_data[:8])
+ if crc != zlib.crc32(decompressed):
+ raise BadGzipFile("CRC check failed")
+ if length != (len(decompressed) & 0xffffffff):
+ raise BadGzipFile("Incorrect length of data produced")
+ decompressed_members.append(decompressed)
+ data = do.unused_data[8:].lstrip(b"\x00")
def main():
diff --git a/common/py3-stdlib/hashlib.py b/common/py3-stdlib/hashlib.py
index 21a73f3..b546a3f 100644
--- a/common/py3-stdlib/hashlib.py
+++ b/common/py3-stdlib/hashlib.py
@@ -65,7 +65,7 @@
algorithms_available = set(__always_supported)
__all__ = __always_supported + ('new', 'algorithms_guaranteed',
- 'algorithms_available', 'pbkdf2_hmac')
+ 'algorithms_available', 'pbkdf2_hmac', 'file_digest')
__builtin_constructor_cache = {}
@@ -235,15 +235,15 @@
loop = 1
from_bytes = int.from_bytes
while len(dkey) < dklen:
- prev = prf(salt + loop.to_bytes(4, 'big'))
+ prev = prf(salt + loop.to_bytes(4))
# endianness doesn't matter here as long to / from use the same
- rkey = int.from_bytes(prev, 'big')
+ rkey = from_bytes(prev)
for i in range(iterations - 1):
prev = prf(prev)
# rkey = rkey ^ prev
- rkey ^= from_bytes(prev, 'big')
+ rkey ^= from_bytes(prev)
loop += 1
- dkey += rkey.to_bytes(inner.digest_size, 'big')
+ dkey += rkey.to_bytes(inner.digest_size)
return dkey[:dklen]
@@ -254,6 +254,52 @@
pass
+def file_digest(fileobj, digest, /, *, _bufsize=2**18):
+ """Hash the contents of a file-like object. Returns a digest object.
+
+ *fileobj* must be a file-like object opened for reading in binary mode.
+ It accepts file objects from open(), io.BytesIO(), and SocketIO objects.
+ The function may bypass Python's I/O and use the file descriptor *fileno*
+ directly.
+
+ *digest* must either be a hash algorithm name as a *str*, a hash
+ constructor, or a callable that returns a hash object.
+ """
+ # On Linux we could use AF_ALG sockets and sendfile() to archive zero-copy
+ # hashing with hardware acceleration.
+ if isinstance(digest, str):
+ digestobj = new(digest)
+ else:
+ digestobj = digest()
+
+ if hasattr(fileobj, "getbuffer"):
+ # io.BytesIO object, use zero-copy buffer
+ digestobj.update(fileobj.getbuffer())
+ return digestobj
+
+ # Only binary files implement readinto().
+ if not (
+ hasattr(fileobj, "readinto")
+ and hasattr(fileobj, "readable")
+ and fileobj.readable()
+ ):
+ raise ValueError(
+ f"'{fileobj!r}' is not a file-like object in binary reading mode."
+ )
+
+ # binary file, socket.SocketIO object
+ # Note: socket I/O uses different syscalls than file I/O.
+ buf = bytearray(_bufsize) # Reusable buffer to reduce allocations.
+ view = memoryview(buf)
+ while True:
+ size = fileobj.readinto(buf)
+ if size == 0:
+ break # EOF
+ digestobj.update(view[:size])
+
+ return digestobj
+
+
for __func_name in __always_supported:
# try them all, some may not work due to the OpenSSL
# version not supporting that algorithm.
diff --git a/common/py3-stdlib/heapq.py b/common/py3-stdlib/heapq.py
index fabefd8..2fd9d1f 100644
--- a/common/py3-stdlib/heapq.py
+++ b/common/py3-stdlib/heapq.py
@@ -12,6 +12,8 @@
item = heappop(heap) # pops the smallest item from the heap
item = heap[0] # smallest item on the heap without popping it
heapify(x) # transforms list into a heap, in-place, in linear time
+item = heappushpop(heap, item) # pushes a new item and then returns
+ # the smallest item; the heap size is unchanged
item = heapreplace(heap, item) # pops and returns smallest item, and adds
# new item; the heap size is unchanged
diff --git a/common/py3-stdlib/html/parser.py b/common/py3-stdlib/html/parser.py
index 58f6bb3..bef0f4f 100644
--- a/common/py3-stdlib/html/parser.py
+++ b/common/py3-stdlib/html/parser.py
@@ -328,13 +328,6 @@
end = rawdata[k:endpos].strip()
if end not in (">", "/>"):
- lineno, offset = self.getpos()
- if "\n" in self.__starttag_text:
- lineno = lineno + self.__starttag_text.count("\n")
- offset = len(self.__starttag_text) \
- - self.__starttag_text.rfind("\n")
- else:
- offset = offset + len(self.__starttag_text)
self.handle_data(rawdata[i:endpos])
return endpos
if end.endswith('/>'):
diff --git a/common/py3-stdlib/http/__init__.py b/common/py3-stdlib/http/__init__.py
index bf8d7d6..cd2885d 100644
--- a/common/py3-stdlib/http/__init__.py
+++ b/common/py3-stdlib/http/__init__.py
@@ -1,9 +1,10 @@
-from enum import IntEnum
+from enum import StrEnum, IntEnum, _simple_enum
-__all__ = ['HTTPStatus']
+__all__ = ['HTTPStatus', 'HTTPMethod']
-class HTTPStatus(IntEnum):
+@_simple_enum(IntEnum)
+class HTTPStatus:
"""HTTP status codes and reason phrases
Status codes from the following RFCs are all observed:
@@ -148,3 +149,32 @@
NETWORK_AUTHENTICATION_REQUIRED = (511,
'Network Authentication Required',
'The client needs to authenticate to gain network access')
+
+
+@_simple_enum(StrEnum)
+class HTTPMethod:
+ """HTTP methods and descriptions
+
+ Methods from the following RFCs are all observed:
+
+ * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616
+ * RFC 5789: PATCH Method for HTTP
+ """
+ def __new__(cls, value, description):
+ obj = str.__new__(cls, value)
+ obj._value_ = value
+ obj.description = description
+ return obj
+
+ def __repr__(self):
+ return "<%s.%s>" % (self.__class__.__name__, self._name_)
+
+ CONNECT = 'CONNECT', 'Establish a connection to the server.'
+ DELETE = 'DELETE', 'Remove the target.'
+ GET = 'GET', 'Retrieve the target.'
+ HEAD = 'HEAD', 'Same as GET, but only retrieve the status line and header section.'
+ OPTIONS = 'OPTIONS', 'Describe the communication options for the target.'
+ PATCH = 'PATCH', 'Apply partial modifications to a target.'
+ POST = 'POST', 'Perform target-specific processing with the request payload.'
+ PUT = 'PUT', 'Replace the target with the request payload.'
+ TRACE = 'TRACE', 'Perform a message loop-back test along the path to the target.'
diff --git a/common/py3-stdlib/http/client.py b/common/py3-stdlib/http/client.py
index d1b7b10..1c7f463 100644
--- a/common/py3-stdlib/http/client.py
+++ b/common/py3-stdlib/http/client.py
@@ -594,8 +594,8 @@
amt -= chunk_left
self.chunk_left = 0
return b''.join(value)
- except IncompleteRead:
- raise IncompleteRead(b''.join(value))
+ except IncompleteRead as exc:
+ raise IncompleteRead(b''.join(value)) from exc
def _readinto_chunked(self, b):
assert self.chunked != _UNKNOWN
@@ -918,23 +918,26 @@
del headers
response = self.response_class(self.sock, method=self._method)
- (version, code, message) = response._read_status()
+ try:
+ (version, code, message) = response._read_status()
- if code != http.HTTPStatus.OK:
- self.close()
- raise OSError(f"Tunnel connection failed: {code} {message.strip()}")
- while True:
- line = response.fp.readline(_MAXLINE + 1)
- if len(line) > _MAXLINE:
- raise LineTooLong("header line")
- if not line:
- # for sites which EOF without sending a trailer
- break
- if line in (b'\r\n', b'\n', b''):
- break
+ if code != http.HTTPStatus.OK:
+ self.close()
+ raise OSError(f"Tunnel connection failed: {code} {message.strip()}")
+ while True:
+ line = response.fp.readline(_MAXLINE + 1)
+ if len(line) > _MAXLINE:
+ raise LineTooLong("header line")
+ if not line:
+ # for sites which EOF without sending a trailer
+ break
+ if line in (b'\r\n', b'\n', b''):
+ break
- if self.debuglevel > 0:
- print('header:', line.decode())
+ if self.debuglevel > 0:
+ print('header:', line.decode())
+ finally:
+ response.close()
def connect(self):
"""Connect to the host and port specified in __init__."""
@@ -981,7 +984,7 @@
print("send:", repr(data))
if hasattr(data, "read") :
if self.debuglevel > 0:
- print("sendIng a read()able")
+ print("sending a readable")
encode = self._is_textIO(data)
if encode and self.debuglevel > 0:
print("encoding file using iso-8859-1")
@@ -1014,7 +1017,7 @@
def _read_readable(self, readable):
if self.debuglevel > 0:
- print("sendIng a read()able")
+ print("reading a readable")
encode = self._is_textIO(readable)
if encode and self.debuglevel > 0:
print("encoding file using iso-8859-1")
diff --git a/common/py3-stdlib/http/cookiejar.py b/common/py3-stdlib/http/cookiejar.py
index 0380d9f..e622fc3 100644
--- a/common/py3-stdlib/http/cookiejar.py
+++ b/common/py3-stdlib/http/cookiejar.py
@@ -89,8 +89,7 @@
DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
-MONTHS_LOWER = []
-for month in MONTHS: MONTHS_LOWER.append(month.lower())
+MONTHS_LOWER = [month.lower() for month in MONTHS]
def time2isoz(t=None):
"""Return a string representing time in seconds since epoch, t.
@@ -1044,12 +1043,13 @@
else:
undotted_domain = domain
embedded_dots = (undotted_domain.find(".") >= 0)
- if not embedded_dots and domain != ".local":
+ if not embedded_dots and not erhn.endswith(".local"):
_debug(" non-local domain %s contains no embedded dot",
domain)
return False
if cookie.version == 0:
- if (not erhn.endswith(domain) and
+ if (not (erhn.endswith(domain) or
+ erhn.endswith(f"{undotted_domain}.local")) and
(not erhn.startswith(".") and
not ("."+erhn).endswith(domain))):
_debug(" effective request-host %s (even with added "
@@ -1224,14 +1224,9 @@
_debug(" %s does not path-match %s", req_path, path)
return False
-def vals_sorted_by_key(adict):
- keys = sorted(adict.keys())
- return map(adict.get, keys)
-
def deepvalues(mapping):
- """Iterates over nested mapping, depth-first, in sorted order by key."""
- values = vals_sorted_by_key(mapping)
- for obj in values:
+ """Iterates over nested mapping, depth-first"""
+ for obj in list(mapping.values()):
mapping = False
try:
obj.items
@@ -1895,7 +1890,10 @@
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
- with open(filename, "w") as f:
+ with os.fdopen(
+ os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600),
+ 'w',
+ ) as f:
# There really isn't an LWP Cookies 2.0 format, but this indicates
# that there is extra information in here (domain_dot and
# port_spec) while still being compatible with libwww-perl, I hope.
@@ -2091,7 +2089,10 @@
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
- with open(filename, "w") as f:
+ with os.fdopen(
+ os.open(filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600),
+ 'w',
+ ) as f:
f.write(NETSCAPE_HEADER_TEXT)
now = time.time()
for cookie in self:
diff --git a/common/py3-stdlib/http/server.py b/common/py3-stdlib/http/server.py
index 9c218d0..da07f11 100644
--- a/common/py3-stdlib/http/server.py
+++ b/common/py3-stdlib/http/server.py
@@ -110,11 +110,10 @@
# Default error message template
DEFAULT_ERROR_MESSAGE = """\
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
- "http://www.w3.org/TR/html4/strict.dtd">
-<html>
+<!DOCTYPE HTML>
+<html lang="en">
<head>
- <meta http-equiv="Content-Type" content="text/html;charset=utf-8">
+ <meta charset="utf-8">
<title>Error response</title>
</head>
<body>
@@ -301,6 +300,10 @@
# - Leading zeros MUST be ignored by recipients.
if len(version_number) != 2:
raise ValueError
+ if any(not component.isdigit() for component in version_number):
+ raise ValueError("non digit in http version")
+ if any(len(component) > 10 for component in version_number):
+ raise ValueError("unreasonable length http version")
version_number = int(version_number[0]), int(version_number[1])
except (ValueError, IndexError):
self.send_error(
@@ -719,7 +722,7 @@
# The test for this was added in test_httpserver.py
# However, some OS platforms accept a trailingSlash as a filename
# See discussion on python-dev and Issue34711 regarding
- # parseing and rejection of filenames with a trailing slash
+ # parsing and rejection of filenames with a trailing slash
if path.endswith("/"):
self.send_error(HTTPStatus.NOT_FOUND, "File not found")
return None
@@ -791,17 +794,16 @@
displaypath = urllib.parse.unquote(self.path,
errors='surrogatepass')
except UnicodeDecodeError:
- displaypath = urllib.parse.unquote(path)
+ displaypath = urllib.parse.unquote(self.path)
displaypath = html.escape(displaypath, quote=False)
enc = sys.getfilesystemencoding()
- title = 'Directory listing for %s' % displaypath
- r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
- '"http://www.w3.org/TR/html4/strict.dtd">')
- r.append('<html>\n<head>')
- r.append('<meta http-equiv="Content-Type" '
- 'content="text/html; charset=%s">' % enc)
- r.append('<title>%s</title>\n</head>' % title)
- r.append('<body>\n<h1>%s</h1>' % title)
+ title = f'Directory listing for {displaypath}'
+ r.append('<!DOCTYPE HTML>')
+ r.append('<html lang="en">')
+ r.append('<head>')
+ r.append(f'<meta charset="{enc}">')
+ r.append(f'<title>{title}</title>\n</head>')
+ r.append(f'<body>\n<h1>{title}</h1>')
r.append('<hr>\n<ul>')
for name in list:
fullname = os.path.join(path, name)
@@ -1275,15 +1277,19 @@
parser = argparse.ArgumentParser()
parser.add_argument('--cgi', action='store_true',
help='run as CGI server')
- parser.add_argument('--bind', '-b', metavar='ADDRESS',
- help='specify alternate bind address '
+ parser.add_argument('-b', '--bind', metavar='ADDRESS',
+ help='bind to this address '
'(default: all interfaces)')
- parser.add_argument('--directory', '-d', default=os.getcwd(),
- help='specify alternate directory '
+ parser.add_argument('-d', '--directory', default=os.getcwd(),
+ help='serve this directory '
'(default: current directory)')
- parser.add_argument('port', action='store', default=8000, type=int,
- nargs='?',
- help='specify alternate port (default: 8000)')
+ parser.add_argument('-p', '--protocol', metavar='VERSION',
+ default='HTTP/1.0',
+ help='conform to this HTTP version '
+ '(default: %(default)s)')
+ parser.add_argument('port', default=8000, type=int, nargs='?',
+ help='bind to this port '
+ '(default: %(default)s)')
args = parser.parse_args()
if args.cgi:
handler_class = CGIHTTPRequestHandler
@@ -1309,4 +1315,5 @@
ServerClass=DualStackServer,
port=args.port,
bind=args.bind,
+ protocol=args.protocol,
)
diff --git a/common/py3-stdlib/imaplib.py b/common/py3-stdlib/imaplib.py
index 7318439..fa4c0f8 100644
--- a/common/py3-stdlib/imaplib.py
+++ b/common/py3-stdlib/imaplib.py
@@ -145,7 +145,7 @@
the global default socket timeout is used
All IMAP4rev1 commands are supported by methods of the same
- name (in lower-case).
+ name (in lowercase).
All arguments to commands are converted to strings, except for
AUTHENTICATE, and the last argument to APPEND which is passed as
diff --git a/common/py3-stdlib/imghdr.py b/common/py3-stdlib/imghdr.py
index 6e01fd8..6a372e6 100644
--- a/common/py3-stdlib/imghdr.py
+++ b/common/py3-stdlib/imghdr.py
@@ -1,9 +1,14 @@
"""Recognize image file formats based on their first few bytes."""
from os import PathLike
+import warnings
__all__ = ["what"]
+
+warnings._deprecated(__name__, remove=(3, 13))
+
+
#-------------------------#
# Recognize image headers #
#-------------------------#
@@ -35,9 +40,11 @@
tests = []
def test_jpeg(h, f):
- """JPEG data in JFIF or Exif format"""
+ """JPEG data with JFIF or Exif markers; and raw JPEG"""
if h[6:10] in (b'JFIF', b'Exif'):
return 'jpeg'
+ elif h[:4] == b'\xff\xd8\xff\xdb':
+ return 'jpeg'
tests.append(test_jpeg)
diff --git a/common/py3-stdlib/imp.py b/common/py3-stdlib/imp.py
index e02aaef..fc42c15 100644
--- a/common/py3-stdlib/imp.py
+++ b/common/py3-stdlib/imp.py
@@ -9,7 +9,7 @@
from _imp import (lock_held, acquire_lock, release_lock,
get_frozen_object, is_frozen_package,
init_frozen, is_builtin, is_frozen,
- _fix_co_filename)
+ _fix_co_filename, _frozen_module_names)
try:
from _imp import create_dynamic
except ImportError:
@@ -226,7 +226,7 @@
"""
suffix, mode, type_ = details
- if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
+ if mode and (not mode.startswith('r') or '+' in mode):
raise ValueError('invalid file open mode {!r}'.format(mode))
elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
msg = 'file object required for import (type code {})'.format(type_)
diff --git a/common/py3-stdlib/importlib/_adapters.py b/common/py3-stdlib/importlib/_adapters.py
deleted file mode 100644
index e72edd1..0000000
--- a/common/py3-stdlib/importlib/_adapters.py
+++ /dev/null
@@ -1,83 +0,0 @@
-from contextlib import suppress
-
-from . import abc
-
-
-class SpecLoaderAdapter:
- """
- Adapt a package spec to adapt the underlying loader.
- """
-
- def __init__(self, spec, adapter=lambda spec: spec.loader):
- self.spec = spec
- self.loader = adapter(spec)
-
- def __getattr__(self, name):
- return getattr(self.spec, name)
-
-
-class TraversableResourcesLoader:
- """
- Adapt a loader to provide TraversableResources.
- """
-
- def __init__(self, spec):
- self.spec = spec
-
- def get_resource_reader(self, name):
- return DegenerateFiles(self.spec)._native()
-
-
-class DegenerateFiles:
- """
- Adapter for an existing or non-existant resource reader
- to provide a degenerate .files().
- """
-
- class Path(abc.Traversable):
- def iterdir(self):
- return iter(())
-
- def is_dir(self):
- return False
-
- is_file = exists = is_dir # type: ignore
-
- def joinpath(self, other):
- return DegenerateFiles.Path()
-
- @property
- def name(self):
- return ''
-
- def open(self, mode='rb', *args, **kwargs):
- raise ValueError()
-
- def __init__(self, spec):
- self.spec = spec
-
- @property
- def _reader(self):
- with suppress(AttributeError):
- return self.spec.loader.get_resource_reader(self.spec.name)
-
- def _native(self):
- """
- Return the native reader if it supports files().
- """
- reader = self._reader
- return reader if hasattr(reader, 'files') else self
-
- def __getattr__(self, attr):
- return getattr(self._reader, attr)
-
- def files(self):
- return DegenerateFiles.Path()
-
-
-def wrap_spec(package):
- """
- Construct a package spec with traversable compatibility
- on the spec/loader/reader.
- """
- return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)
diff --git a/common/py3-stdlib/importlib/_bootstrap.py b/common/py3-stdlib/importlib/_bootstrap.py
index 527bc9c..ee93ebc 100644
--- a/common/py3-stdlib/importlib/_bootstrap.py
+++ b/common/py3-stdlib/importlib/_bootstrap.py
@@ -361,6 +361,7 @@
self.origin = origin
self.loader_state = loader_state
self.submodule_search_locations = [] if is_package else None
+ self._uninitialized_submodules = []
# file-location attributes
self._set_fileattr = False
@@ -420,7 +421,10 @@
def spec_from_loader(name, loader, *, origin=None, is_package=None):
"""Return a module spec based on various loader methods."""
- if hasattr(loader, 'get_filename'):
+ if origin is None:
+ origin = getattr(loader, '_ORIGIN', None)
+
+ if not origin and hasattr(loader, 'get_filename'):
if _bootstrap_external is None:
raise NotImplementedError
spec_from_file_location = _bootstrap_external.spec_from_file_location
@@ -466,12 +470,9 @@
except AttributeError:
location = None
if origin is None:
- if location is None:
- try:
- origin = loader._ORIGIN
- except AttributeError:
- origin = None
- else:
+ if loader is not None:
+ origin = getattr(loader, '_ORIGIN', None)
+ if not origin and location is not None:
origin = location
try:
cached = module.__cached__
@@ -483,7 +484,7 @@
submodule_search_locations = None
spec = ModuleSpec(name, loader, origin=origin)
- spec._set_fileattr = False if location is None else True
+ spec._set_fileattr = False if location is None else (origin == location)
spec.cached = cached
spec.submodule_search_locations = submodule_search_locations
return spec
@@ -506,9 +507,9 @@
if spec.submodule_search_locations is not None:
if _bootstrap_external is None:
raise NotImplementedError
- _NamespaceLoader = _bootstrap_external._NamespaceLoader
+ NamespaceLoader = _bootstrap_external.NamespaceLoader
- loader = _NamespaceLoader.__new__(_NamespaceLoader)
+ loader = NamespaceLoader.__new__(NamespaceLoader)
loader._path = spec.submodule_search_locations
spec.loader = loader
# While the docs say that module.__file__ is not set for
@@ -540,6 +541,7 @@
# __path__
if override or getattr(module, '__path__', None) is None:
if spec.submodule_search_locations is not None:
+ # XXX We should extend __path__ if it's already a list.
try:
module.__path__ = spec.submodule_search_locations
except AttributeError:
@@ -745,8 +747,6 @@
@classmethod
def find_spec(cls, fullname, path=None, target=None):
- if path is not None:
- return None
if _imp.is_builtin(fullname):
return spec_from_loader(fullname, cls, origin=cls._ORIGIN)
else:
@@ -824,11 +824,128 @@
return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
@classmethod
- def find_spec(cls, fullname, path=None, target=None):
- if _imp.is_frozen(fullname):
- return spec_from_loader(fullname, cls, origin=cls._ORIGIN)
+ def _fix_up_module(cls, module):
+ spec = module.__spec__
+ state = spec.loader_state
+ if state is None:
+ # The module is missing FrozenImporter-specific values.
+
+ # Fix up the spec attrs.
+ origname = vars(module).pop('__origname__', None)
+ assert origname, 'see PyImport_ImportFrozenModuleObject()'
+ ispkg = hasattr(module, '__path__')
+ assert _imp.is_frozen_package(module.__name__) == ispkg, ispkg
+ filename, pkgdir = cls._resolve_filename(origname, spec.name, ispkg)
+ spec.loader_state = type(sys.implementation)(
+ filename=filename,
+ origname=origname,
+ )
+ __path__ = spec.submodule_search_locations
+ if ispkg:
+ assert __path__ == [], __path__
+ if pkgdir:
+ spec.submodule_search_locations.insert(0, pkgdir)
+ else:
+ assert __path__ is None, __path__
+
+ # Fix up the module attrs (the bare minimum).
+ assert not hasattr(module, '__file__'), module.__file__
+ if filename:
+ try:
+ module.__file__ = filename
+ except AttributeError:
+ pass
+ if ispkg:
+ if module.__path__ != __path__:
+ assert module.__path__ == [], module.__path__
+ module.__path__.extend(__path__)
else:
+ # These checks ensure that _fix_up_module() is only called
+ # in the right places.
+ __path__ = spec.submodule_search_locations
+ ispkg = __path__ is not None
+ # Check the loader state.
+ assert sorted(vars(state)) == ['filename', 'origname'], state
+ if state.origname:
+ # The only frozen modules with "origname" set are stdlib modules.
+ (__file__, pkgdir,
+ ) = cls._resolve_filename(state.origname, spec.name, ispkg)
+ assert state.filename == __file__, (state.filename, __file__)
+ if pkgdir:
+ assert __path__ == [pkgdir], (__path__, pkgdir)
+ else:
+ assert __path__ == ([] if ispkg else None), __path__
+ else:
+ __file__ = None
+ assert state.filename is None, state.filename
+ assert __path__ == ([] if ispkg else None), __path__
+ # Check the file attrs.
+ if __file__:
+ assert hasattr(module, '__file__')
+ assert module.__file__ == __file__, (module.__file__, __file__)
+ else:
+ assert not hasattr(module, '__file__'), module.__file__
+ if ispkg:
+ assert hasattr(module, '__path__')
+ assert module.__path__ == __path__, (module.__path__, __path__)
+ else:
+ assert not hasattr(module, '__path__'), module.__path__
+ assert not spec.has_location
+
+ @classmethod
+ def _resolve_filename(cls, fullname, alias=None, ispkg=False):
+ if not fullname or not getattr(sys, '_stdlib_dir', None):
+ return None, None
+ try:
+ sep = cls._SEP
+ except AttributeError:
+ sep = cls._SEP = '\\' if sys.platform == 'win32' else '/'
+
+ if fullname != alias:
+ if fullname.startswith('<'):
+ fullname = fullname[1:]
+ if not ispkg:
+ fullname = f'{fullname}.__init__'
+ else:
+ ispkg = False
+ relfile = fullname.replace('.', sep)
+ if ispkg:
+ pkgdir = f'{sys._stdlib_dir}{sep}{relfile}'
+ filename = f'{pkgdir}{sep}__init__.py'
+ else:
+ pkgdir = None
+ filename = f'{sys._stdlib_dir}{sep}{relfile}.py'
+ return filename, pkgdir
+
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ info = _call_with_frames_removed(_imp.find_frozen, fullname)
+ if info is None:
return None
+ # We get the marshaled data in exec_module() (the loader
+ # part of the importer), instead of here (the finder part).
+ # The loader is the usual place to get the data that will
+ # be loaded into the module. (For example, see _LoaderBasics
+ # in _bootstra_external.py.) Most importantly, this importer
+ # is simpler if we wait to get the data.
+ # However, getting as much data in the finder as possible
+ # to later load the module is okay, and sometimes important.
+ # (That's why ModuleSpec.loader_state exists.) This is
+ # especially true if it avoids throwing away expensive data
+ # the loader would otherwise duplicate later and can be done
+ # efficiently. In this case it isn't worth it.
+ _, ispkg, origname = info
+ spec = spec_from_loader(fullname, cls,
+ origin=cls._ORIGIN,
+ is_package=ispkg)
+ filename, pkgdir = cls._resolve_filename(origname, fullname, ispkg)
+ spec.loader_state = type(sys.implementation)(
+ filename=filename,
+ origname=origname,
+ )
+ if pkgdir:
+ spec.submodule_search_locations.insert(0, pkgdir)
+ return spec
@classmethod
def find_module(cls, fullname, path=None):
@@ -844,14 +961,21 @@
@staticmethod
def create_module(spec):
- """Use default semantics for module creation."""
+ """Set __file__, if able."""
+ module = _new_module(spec.name)
+ try:
+ filename = spec.loader_state.filename
+ except AttributeError:
+ pass
+ else:
+ if filename:
+ module.__file__ = filename
+ return module
@staticmethod
def exec_module(module):
- name = module.__spec__.name
- if not _imp.is_frozen(name):
- raise ImportError('{!r} is not a frozen module'.format(name),
- name=name)
+ spec = module.__spec__
+ name = spec.name
code = _call_with_frames_removed(_imp.get_frozen_object, name)
exec(code, module.__dict__)
@@ -863,7 +987,16 @@
"""
# Warning about deprecation implemented in _load_module_shim().
- return _load_module_shim(cls, fullname)
+ module = _load_module_shim(cls, fullname)
+ info = _imp.find_frozen(fullname)
+ assert info is not None
+ _, ispkg, origname = info
+ module.__origname__ = origname
+ vars(module).pop('__file__', None)
+ if ispkg:
+ module.__path__ = []
+ cls._fix_up_module(module)
+ return module
@classmethod
@_requires_frozen
@@ -987,6 +1120,7 @@
def _find_and_load_unlocked(name, import_):
path = None
parent = name.rpartition('.')[0]
+ parent_spec = None
if parent:
if parent not in sys.modules:
_call_with_frames_removed(import_, parent)
@@ -999,15 +1133,24 @@
except AttributeError:
msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent)
raise ModuleNotFoundError(msg, name=name) from None
+ parent_spec = parent_module.__spec__
+ child = name.rpartition('.')[2]
spec = _find_spec(name, path)
if spec is None:
raise ModuleNotFoundError(_ERR_MSG.format(name), name=name)
else:
- module = _load_unlocked(spec)
+ if parent_spec:
+ # Temporarily add child we are currently importing to parent's
+ # _uninitialized_submodules for circular import tracking.
+ parent_spec._uninitialized_submodules.append(child)
+ try:
+ module = _load_unlocked(spec)
+ finally:
+ if parent_spec:
+ parent_spec._uninitialized_submodules.pop()
if parent:
# Set the module as an attribute on its parent.
parent_module = sys.modules[parent]
- child = name.rpartition('.')[2]
try:
setattr(parent_module, child, module)
except AttributeError:
@@ -1021,17 +1164,28 @@
def _find_and_load(name, import_):
"""Find and load the module."""
- with _ModuleLockManager(name):
- module = sys.modules.get(name, _NEEDS_LOADING)
- if module is _NEEDS_LOADING:
- return _find_and_load_unlocked(name, import_)
+
+ # Optimization: we avoid unneeded module locking if the module
+ # already exists in sys.modules and is fully initialized.
+ module = sys.modules.get(name, _NEEDS_LOADING)
+ if (module is _NEEDS_LOADING or
+ getattr(getattr(module, "__spec__", None), "_initializing", False)):
+ with _ModuleLockManager(name):
+ module = sys.modules.get(name, _NEEDS_LOADING)
+ if module is _NEEDS_LOADING:
+ return _find_and_load_unlocked(name, import_)
+
+ # Optimization: only call _bootstrap._lock_unlock_module() if
+ # module.__spec__._initializing is True.
+ # NOTE: because of this, initializing must be set *before*
+ # putting the new module in sys.modules.
+ _lock_unlock_module(name)
if module is None:
message = ('import of {} halted; '
'None in sys.modules'.format(name))
raise ModuleNotFoundError(message, name=name)
- _lock_unlock_module(name)
return module
@@ -1182,6 +1336,8 @@
continue
spec = _spec_from_module(module, loader)
_init_module_attrs(spec, module)
+ if loader is FrozenImporter:
+ loader._fix_up_module(module)
# Directly load built-in modules needed during bootstrap.
self_module = sys.modules[__name__]
diff --git a/common/py3-stdlib/importlib/_bootstrap_external.py b/common/py3-stdlib/importlib/_bootstrap_external.py
index 49bcaea..f603a89 100644
--- a/common/py3-stdlib/importlib/_bootstrap_external.py
+++ b/common/py3-stdlib/importlib/_bootstrap_external.py
@@ -352,16 +352,73 @@
# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
# Python 3.10b1 3438 Safer line number table handling.
# Python 3.10b1 3439 (Add ROT_N)
+# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling)
+# Python 3.11a1 3451 (Add CALL_METHOD_KW)
+# Python 3.11a1 3452 (drop nlocals from marshaled code objects)
+# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
+# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
+# Python 3.11a1 3455 (add MAKE_CELL bpo-43693)
+# Python 3.11a1 3456 (interleave cell args bpo-43693)
+# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693)
+# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD)
+# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions)
+# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530)
+# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards)
+# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change
+# MATCH_CLASS and MATCH_KEYS, and add COPY)
+# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the
+# active exception)
+# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into
+# BINARY_OP)
+# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode)
+# Python 3.11a4 3466 (bpo-45292: PEP-654 except*)
+# Python 3.11a4 3467 (Change CALL_xxx opcodes)
+# Python 3.11a4 3468 (Add SEND opcode)
+# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info)
+# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti)
+# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE)
+# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
+# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
+# Python 3.11a4 3474 (Add RESUME opcode)
+# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode)
+# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode)
+# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and
+# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP)
+# Python 3.11a5 3478 (New CALL opcodes)
+# Python 3.11a5 3479 (Add PUSH_NULL opcode)
+# Python 3.11a5 3480 (New CALL opcodes, second iteration)
+# Python 3.11a5 3481 (Use inline cache for BINARY_OP)
+# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL)
+# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR)
+# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and
+# STORE_ATTR)
+# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE)
+# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL)
+# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism)
+# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL)
+# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE)
+# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH)
+# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH,
+# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual)
+# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative)
+# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative)
+# Python 3.11a7 3494 (New location info table)
+# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626)
+# Python 3.12 will start with magic number 3500
+
#
# MAGIC must change whenever the bytecode emitted by the compiler may no
# longer be understood by older implementations of the eval loop (usually
# due to the addition of new opcodes).
#
+# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n.
+#
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.
-MAGIC_NUMBER = (3439).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3495).to_bytes(2, 'little') + b'\r\n'
+
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
_PYCACHE = '__pycache__'
@@ -1172,7 +1229,7 @@
return hash(self.name) ^ hash(self.path)
def create_module(self, spec):
- """Create an unitialized extension module"""
+ """Create an uninitialized extension module"""
module = _bootstrap._call_with_frames_removed(
_imp.create_dynamic, spec)
_bootstrap._verbose_message('extension module {!r} loaded from {!r}',
@@ -1273,8 +1330,10 @@
self._path.append(item)
-# We use this exclusively in module_from_spec() for backward-compatibility.
-class _NamespaceLoader:
+# This class is actually exposed publicly in a namespace package's __loader__
+# attribute, so it should be available through a non-private name.
+# https://bugs.python.org/issue35673
+class NamespaceLoader:
def __init__(self, name, path, path_finder):
self._path = _NamespacePath(name, path, path_finder)
@@ -1285,7 +1344,7 @@
The method is deprecated. The import machinery does the job itself.
"""
- _warnings.warn("_NamespaceLoader.module_repr() is deprecated and "
+ _warnings.warn("NamespaceLoader.module_repr() is deprecated and "
"slated for removal in Python 3.12", DeprecationWarning)
return '<module {!r} (namespace)>'.format(module.__name__)
@@ -1321,6 +1380,10 @@
return NamespaceReader(self._path)
+# We use this exclusively in module_from_spec() for backward-compatibility.
+_NamespaceLoader = NamespaceLoader
+
+
# Finders #####################################################################
class PathFinder:
@@ -1332,7 +1395,9 @@
"""Call the invalidate_caches() method on all path entry finders
stored in sys.path_importer_caches (where implemented)."""
for name, finder in list(sys.path_importer_cache.items()):
- if finder is None:
+ # Drop entry if finder name is a relative path. The current
+ # working directory may have changed.
+ if finder is None or not _path_isabs(name):
del sys.path_importer_cache[name]
elif hasattr(finder, 'invalidate_caches'):
finder.invalidate_caches()
@@ -1403,7 +1468,7 @@
# the list of paths that will become its __path__
namespace_path = []
for entry in path:
- if not isinstance(entry, (str, bytes)):
+ if not isinstance(entry, str):
continue
finder = cls._path_importer_cache(entry)
if finder is not None:
@@ -1500,9 +1565,12 @@
loaders.extend((suffix, loader) for suffix in suffixes)
self._loaders = loaders
# Base (directory) path
- self.path = path or '.'
- if not _path_isabs(self.path):
- self.path = _path_join(_os.getcwd(), self.path)
+ if not path or path == '.':
+ self.path = _os.getcwd()
+ elif not _path_isabs(path):
+ self.path = _path_join(_os.getcwd(), path)
+ else:
+ self.path = path
self._path_mtime = -1
self._path_cache = set()
self._relaxed_path_cache = set()
diff --git a/common/py3-stdlib/importlib/abc.py b/common/py3-stdlib/importlib/abc.py
index 0b4a3f8..3fa151f 100644
--- a/common/py3-stdlib/importlib/abc.py
+++ b/common/py3-stdlib/importlib/abc.py
@@ -14,8 +14,19 @@
from ._abc import Loader
import abc
import warnings
-from typing import BinaryIO, Iterable, Text
-from typing import Protocol, runtime_checkable
+
+# for compatibility with Python 3.10
+from .resources.abc import ResourceReader, Traversable, TraversableResources
+
+
+__all__ = [
+ 'Loader', 'Finder', 'MetaPathFinder', 'PathEntryFinder',
+ 'ResourceLoader', 'InspectLoader', 'ExecutionLoader',
+ 'FileLoader', 'SourceLoader',
+
+ # for compatibility with Python 3.10
+ 'ResourceReader', 'Traversable', 'TraversableResources',
+]
def _register(abstract_cls, *classes):
@@ -213,7 +224,7 @@
exec_module = _bootstrap_external._LoaderBasics.exec_module
load_module = _bootstrap_external._LoaderBasics.load_module
-_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter)
+_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.NamespaceLoader)
class ExecutionLoader(InspectLoader):
@@ -307,136 +318,3 @@
"""
_register(SourceLoader, machinery.SourceFileLoader)
-
-
-class ResourceReader(metaclass=abc.ABCMeta):
- """Abstract base class for loaders to provide resource reading support."""
-
- @abc.abstractmethod
- def open_resource(self, resource: Text) -> BinaryIO:
- """Return an opened, file-like object for binary reading.
-
- The 'resource' argument is expected to represent only a file name.
- If the resource cannot be found, FileNotFoundError is raised.
- """
- # This deliberately raises FileNotFoundError instead of
- # NotImplementedError so that if this method is accidentally called,
- # it'll still do the right thing.
- raise FileNotFoundError
-
- @abc.abstractmethod
- def resource_path(self, resource: Text) -> Text:
- """Return the file system path to the specified resource.
-
- The 'resource' argument is expected to represent only a file name.
- If the resource does not exist on the file system, raise
- FileNotFoundError.
- """
- # This deliberately raises FileNotFoundError instead of
- # NotImplementedError so that if this method is accidentally called,
- # it'll still do the right thing.
- raise FileNotFoundError
-
- @abc.abstractmethod
- def is_resource(self, path: Text) -> bool:
- """Return True if the named 'path' is a resource.
-
- Files are resources, directories are not.
- """
- raise FileNotFoundError
-
- @abc.abstractmethod
- def contents(self) -> Iterable[str]:
- """Return an iterable of entries in `package`."""
- raise FileNotFoundError
-
-
-@runtime_checkable
-class Traversable(Protocol):
- """
- An object with a subset of pathlib.Path methods suitable for
- traversing directories and opening files.
- """
-
- @abc.abstractmethod
- def iterdir(self):
- """
- Yield Traversable objects in self
- """
-
- def read_bytes(self):
- """
- Read contents of self as bytes
- """
- with self.open('rb') as strm:
- return strm.read()
-
- def read_text(self, encoding=None):
- """
- Read contents of self as text
- """
- with self.open(encoding=encoding) as strm:
- return strm.read()
-
- @abc.abstractmethod
- def is_dir(self) -> bool:
- """
- Return True if self is a dir
- """
-
- @abc.abstractmethod
- def is_file(self) -> bool:
- """
- Return True if self is a file
- """
-
- @abc.abstractmethod
- def joinpath(self, child):
- """
- Return Traversable child in self
- """
-
- def __truediv__(self, child):
- """
- Return Traversable child in self
- """
- return self.joinpath(child)
-
- @abc.abstractmethod
- def open(self, mode='r', *args, **kwargs):
- """
- mode may be 'r' or 'rb' to open as text or binary. Return a handle
- suitable for reading (same as pathlib.Path.open).
-
- When opening as text, accepts encoding parameters such as those
- accepted by io.TextIOWrapper.
- """
-
- @abc.abstractproperty
- def name(self) -> str:
- """
- The base name of this object without any parent references.
- """
-
-
-class TraversableResources(ResourceReader):
- """
- The required interface for providing traversable
- resources.
- """
-
- @abc.abstractmethod
- def files(self):
- """Return a Traversable object for the loaded package."""
-
- def open_resource(self, resource):
- return self.files().joinpath(resource).open('rb')
-
- def resource_path(self, resource):
- raise FileNotFoundError(resource)
-
- def is_resource(self, path):
- return self.files().joinpath(path).is_file()
-
- def contents(self):
- return (item.name for item in self.files().iterdir())
diff --git a/common/py3-stdlib/importlib/machinery.py b/common/py3-stdlib/importlib/machinery.py
index 9a7757f..d9a19a1 100644
--- a/common/py3-stdlib/importlib/machinery.py
+++ b/common/py3-stdlib/importlib/machinery.py
@@ -12,6 +12,7 @@
from ._bootstrap_external import SourceFileLoader
from ._bootstrap_external import SourcelessFileLoader
from ._bootstrap_external import ExtensionFileLoader
+from ._bootstrap_external import NamespaceLoader
def all_suffixes():
diff --git a/common/py3-stdlib/importlib/metadata/__init__.py b/common/py3-stdlib/importlib/metadata/__init__.py
index 682067d..bbdbcee 100644
--- a/common/py3-stdlib/importlib/metadata/__init__.py
+++ b/common/py3-stdlib/importlib/metadata/__init__.py
@@ -15,10 +15,9 @@
import collections
from . import _adapters, _meta
-from ._meta import PackageMetadata
from ._collections import FreezableDefaultDict, Pair
from ._functools import method_cache, pass_none
-from ._itertools import unique_everseen
+from ._itertools import always_iterable, unique_everseen
from ._meta import PackageMetadata, SimplePath
from contextlib import suppress
@@ -121,8 +120,33 @@
return line and not line.startswith('#')
-class EntryPoint(
- collections.namedtuple('EntryPointBase', 'name value group')):
+class DeprecatedTuple:
+ """
+ Provide subscript item access for backward compatibility.
+
+ >>> recwarn = getfixture('recwarn')
+ >>> ep = EntryPoint(name='name', value='value', group='group')
+ >>> ep[:]
+ ('name', 'value', 'group')
+ >>> ep[0]
+ 'name'
+ >>> len(recwarn)
+ 1
+ """
+
+ _warn = functools.partial(
+ warnings.warn,
+ "EntryPoint tuple interface is deprecated. Access members by name.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ def __getitem__(self, item):
+ self._warn()
+ return self._key()[item]
+
+
+class EntryPoint(DeprecatedTuple):
"""An entry point as defined by Python packaging conventions.
See `the packaging docs on entry points
@@ -160,8 +184,15 @@
following the attr, and following any extras.
"""
+ name: str
+ value: str
+ group: str
+
dist: Optional['Distribution'] = None
+ def __init__(self, name, value, group):
+ vars(self).update(name=name, value=value, group=group)
+
def load(self):
"""Load the entry point from its definition. If only a module
is indicated by the value, return that module. Otherwise,
@@ -188,7 +219,7 @@
return re.findall(r'\w+', match.group('extras') or '')
def _for(self, dist):
- self.dist = dist
+ vars(self).update(dist=dist)
return self
def __iter__(self):
@@ -202,12 +233,6 @@
warnings.warn(msg, DeprecationWarning)
return iter((self.name, self))
- def __reduce__(self):
- return (
- self.__class__,
- (self.name, self.value, self.group),
- )
-
def matches(self, **params):
"""
EntryPoint matches the given parameters.
@@ -231,6 +256,27 @@
attrs = (getattr(self, param) for param in params)
return all(map(operator.eq, params.values(), attrs))
+ def _key(self):
+ return self.name, self.value, self.group
+
+ def __lt__(self, other):
+ return self._key() < other._key()
+
+ def __eq__(self, other):
+ return self._key() == other._key()
+
+ def __setattr__(self, name, value):
+ raise AttributeError("EntryPoint objects are immutable.")
+
+ def __repr__(self):
+ return (
+ f'EntryPoint(name={self.name!r}, value={self.value!r}, '
+ f'group={self.group!r})'
+ )
+
+ def __hash__(self):
+ return hash(self._key())
+
class DeprecatedList(list):
"""
@@ -273,37 +319,20 @@
stacklevel=2,
)
- def __setitem__(self, *args, **kwargs):
- self._warn()
- return super().__setitem__(*args, **kwargs)
+ def _wrap_deprecated_method(method_name: str): # type: ignore
+ def wrapped(self, *args, **kwargs):
+ self._warn()
+ return getattr(super(), method_name)(*args, **kwargs)
- def __delitem__(self, *args, **kwargs):
- self._warn()
- return super().__delitem__(*args, **kwargs)
+ return method_name, wrapped
- def append(self, *args, **kwargs):
- self._warn()
- return super().append(*args, **kwargs)
-
- def reverse(self, *args, **kwargs):
- self._warn()
- return super().reverse(*args, **kwargs)
-
- def extend(self, *args, **kwargs):
- self._warn()
- return super().extend(*args, **kwargs)
-
- def pop(self, *args, **kwargs):
- self._warn()
- return super().pop(*args, **kwargs)
-
- def remove(self, *args, **kwargs):
- self._warn()
- return super().remove(*args, **kwargs)
-
- def __iadd__(self, *args, **kwargs):
- self._warn()
- return super().__iadd__(*args, **kwargs)
+ locals().update(
+ map(
+ _wrap_deprecated_method,
+ '__setitem__ __delitem__ append reverse extend pop remove '
+ '__iadd__ insert sort'.split(),
+ )
+ )
def __add__(self, other):
if not isinstance(other, tuple):
@@ -311,14 +340,6 @@
other = tuple(other)
return self.__class__(tuple(self) + other)
- def insert(self, *args, **kwargs):
- self._warn()
- return super().insert(*args, **kwargs)
-
- def sort(self, *args, **kwargs):
- self._warn()
- return super().sort(*args, **kwargs)
-
def __eq__(self, other):
if not isinstance(other, tuple):
self._warn()
@@ -363,7 +384,7 @@
"""
Return the set of all names of all entry points.
"""
- return set(ep.name for ep in self)
+ return {ep.name for ep in self}
@property
def groups(self):
@@ -374,21 +395,17 @@
>>> EntryPoints().groups
set()
"""
- return set(ep.group for ep in self)
+ return {ep.group for ep in self}
@classmethod
def _from_text_for(cls, text, dist):
return cls(ep._for(dist) for ep in cls._from_text(text))
- @classmethod
- def _from_text(cls, text):
- return itertools.starmap(EntryPoint, cls._parse_groups(text or ''))
-
@staticmethod
- def _parse_groups(text):
+ def _from_text(text):
return (
- (item.value.name, item.value.value, item.name)
- for item in Sectioned.section_pairs(text)
+ EntryPoint(name=item.value.name, value=item.value.value, group=item.name)
+ for item in Sectioned.section_pairs(text or '')
)
@@ -530,7 +547,7 @@
"""
@classmethod
- def from_name(cls, name):
+ def from_name(cls, name: str):
"""Return the Distribution for the given package name.
:param name: The name of the distribution package to search for.
@@ -538,13 +555,13 @@
package, if found.
:raises PackageNotFoundError: When the named package's distribution
metadata cannot be found.
+ :raises ValueError: When an invalid value is supplied for name.
"""
- for resolver in cls._discover_resolvers():
- dists = resolver(DistributionFinder.Context(name=name))
- dist = next(iter(dists), None)
- if dist is not None:
- return dist
- else:
+ if not name:
+ raise ValueError("A distribution name is required.")
+ try:
+ return next(cls.discover(name=name))
+ except StopIteration:
raise PackageNotFoundError(name)
@classmethod
@@ -582,18 +599,6 @@
)
return filter(None, declared)
- @classmethod
- def _local(cls, root='.'):
- from pep517 import build, meta
-
- system = build.compat_system(root)
- builder = functools.partial(
- meta.build,
- source_dir=root,
- system=system,
- )
- return PathDistribution(zipfile.Path(meta.build_as_zip(builder)))
-
@property
def metadata(self) -> _meta.PackageMetadata:
"""Return the parsed metadata for this Distribution.
@@ -641,7 +646,6 @@
missing.
Result may be empty if the metadata exists but is empty.
"""
- file_lines = self._read_files_distinfo() or self._read_files_egginfo()
def make_file(name, hash=None, size_str=None):
result = PackagePath(name)
@@ -650,7 +654,11 @@
result.dist = self
return result
- return file_lines and list(starmap(make_file, csv.reader(file_lines)))
+ @pass_none
+ def make_files(lines):
+ return list(starmap(make_file, csv.reader(lines)))
+
+ return make_files(self._read_files_distinfo() or self._read_files_egginfo())
def _read_files_distinfo(self):
"""
@@ -678,7 +686,7 @@
def _read_egg_info_reqs(self):
source = self.read_text('requires.txt')
- return None if source is None else self._deps_from_requires_text(source)
+ return pass_none(self._deps_from_requires_text)(source)
@classmethod
def _deps_from_requires_text(cls, source):
@@ -772,6 +780,9 @@
"""
Micro-optimized class for searching a path for
children.
+
+ >>> FastPath('').children()
+ ['...']
"""
@functools.lru_cache() # type: ignore
@@ -952,6 +963,7 @@
'CherryPy'
>>> PathDistribution._name_from_stem('face.egg-info')
'face'
+ >>> PathDistribution._name_from_stem('foo.bar')
"""
filename, ext = os.path.splitext(stem)
if ext not in ('.dist-info', '.egg-info'):
@@ -996,6 +1008,15 @@
return distribution(distribution_name).version
+_unique = functools.partial(
+ unique_everseen,
+ key=operator.attrgetter('_normalized_name'),
+)
+"""
+Wrapper for ``distributions`` to return unique distributions by name.
+"""
+
+
def entry_points(**params) -> Union[EntryPoints, SelectableGroups]:
"""Return EntryPoint objects for all installed packages.
@@ -1013,10 +1034,8 @@
:return: EntryPoints or SelectableGroups for all installed packages.
"""
- norm_name = operator.attrgetter('_normalized_name')
- unique = functools.partial(unique_everseen, key=norm_name)
eps = itertools.chain.from_iterable(
- dist.entry_points for dist in unique(distributions())
+ dist.entry_points for dist in _unique(distributions())
)
return SelectableGroups.load(eps).select(**params)
@@ -1052,6 +1071,18 @@
"""
pkg_to_dist = collections.defaultdict(list)
for dist in distributions():
- for pkg in (dist.read_text('top_level.txt') or '').split():
+ for pkg in _top_level_declared(dist) or _top_level_inferred(dist):
pkg_to_dist[pkg].append(dist.metadata['Name'])
return dict(pkg_to_dist)
+
+
+def _top_level_declared(dist):
+ return (dist.read_text('top_level.txt') or '').split()
+
+
+def _top_level_inferred(dist):
+ return {
+ f.parts[0] if len(f.parts) > 1 else f.with_suffix('').name
+ for f in always_iterable(dist.files)
+ if f.suffix == ".py"
+ }
diff --git a/common/py3-stdlib/importlib/metadata/_itertools.py b/common/py3-stdlib/importlib/metadata/_itertools.py
index dd45f2f..d4ca9b9 100644
--- a/common/py3-stdlib/importlib/metadata/_itertools.py
+++ b/common/py3-stdlib/importlib/metadata/_itertools.py
@@ -17,3 +17,57 @@
if k not in seen:
seen_add(k)
yield element
+
+
+# copied from more_itertools 8.8
+def always_iterable(obj, base_type=(str, bytes)):
+ """If *obj* is iterable, return an iterator over its items::
+
+ >>> obj = (1, 2, 3)
+ >>> list(always_iterable(obj))
+ [1, 2, 3]
+
+ If *obj* is not iterable, return a one-item iterable containing *obj*::
+
+ >>> obj = 1
+ >>> list(always_iterable(obj))
+ [1]
+
+ If *obj* is ``None``, return an empty iterable:
+
+ >>> obj = None
+ >>> list(always_iterable(None))
+ []
+
+ By default, binary and text strings are not considered iterable::
+
+ >>> obj = 'foo'
+ >>> list(always_iterable(obj))
+ ['foo']
+
+ If *base_type* is set, objects for which ``isinstance(obj, base_type)``
+ returns ``True`` won't be considered iterable.
+
+ >>> obj = {'a': 1}
+ >>> list(always_iterable(obj)) # Iterate over the dict's keys
+ ['a']
+ >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit
+ [{'a': 1}]
+
+ Set *base_type* to ``None`` to avoid any special handling and treat objects
+ Python considers iterable as iterable:
+
+ >>> obj = 'foo'
+ >>> list(always_iterable(obj, base_type=None))
+ ['f', 'o', 'o']
+ """
+ if obj is None:
+ return iter(())
+
+ if (base_type is not None) and isinstance(obj, base_type):
+ return iter((obj,))
+
+ try:
+ return iter(obj)
+ except TypeError:
+ return iter((obj,))
diff --git a/common/py3-stdlib/importlib/metadata/_meta.py b/common/py3-stdlib/importlib/metadata/_meta.py
index 1a6edbf..d5c0576 100644
--- a/common/py3-stdlib/importlib/metadata/_meta.py
+++ b/common/py3-stdlib/importlib/metadata/_meta.py
@@ -37,7 +37,7 @@
def joinpath(self) -> 'SimplePath':
... # pragma: no cover
- def __div__(self) -> 'SimplePath':
+ def __truediv__(self) -> 'SimplePath':
... # pragma: no cover
def parent(self) -> 'SimplePath':
diff --git a/common/py3-stdlib/importlib/metadata/_text.py b/common/py3-stdlib/importlib/metadata/_text.py
index 766979d..c88cfbb 100644
--- a/common/py3-stdlib/importlib/metadata/_text.py
+++ b/common/py3-stdlib/importlib/metadata/_text.py
@@ -80,7 +80,7 @@
return hash(self.lower())
def __contains__(self, other):
- return super(FoldedCase, self).lower().__contains__(other.lower())
+ return super().lower().__contains__(other.lower())
def in_(self, other):
"Does self appear in other?"
@@ -89,7 +89,7 @@
# cache lower since it's likely to be called frequently.
@method_cache
def lower(self):
- return super(FoldedCase, self).lower()
+ return super().lower()
def index(self, sub):
return self.lower().index(sub.lower())
diff --git a/common/py3-stdlib/importlib/readers.py b/common/py3-stdlib/importlib/readers.py
index 41089c0..df7fb92 100644
--- a/common/py3-stdlib/importlib/readers.py
+++ b/common/py3-stdlib/importlib/readers.py
@@ -1,123 +1,12 @@
-import collections
-import zipfile
-import pathlib
-from . import abc
+"""
+Compatibility shim for .resources.readers as found on Python 3.10.
+Consumers that can rely on Python 3.11 should use the other
+module directly.
+"""
-def remove_duplicates(items):
- return iter(collections.OrderedDict.fromkeys(items))
+from .resources.readers import (
+ FileReader, ZipReader, MultiplexedPath, NamespaceReader,
+)
-
-class FileReader(abc.TraversableResources):
- def __init__(self, loader):
- self.path = pathlib.Path(loader.path).parent
-
- def resource_path(self, resource):
- """
- Return the file system path to prevent
- `resources.path()` from creating a temporary
- copy.
- """
- return str(self.path.joinpath(resource))
-
- def files(self):
- return self.path
-
-
-class ZipReader(abc.TraversableResources):
- def __init__(self, loader, module):
- _, _, name = module.rpartition('.')
- self.prefix = loader.prefix.replace('\\', '/') + name + '/'
- self.archive = loader.archive
-
- def open_resource(self, resource):
- try:
- return super().open_resource(resource)
- except KeyError as exc:
- raise FileNotFoundError(exc.args[0])
-
- def is_resource(self, path):
- # workaround for `zipfile.Path.is_file` returning true
- # for non-existent paths.
- target = self.files().joinpath(path)
- return target.is_file() and target.exists()
-
- def files(self):
- return zipfile.Path(self.archive, self.prefix)
-
-
-class MultiplexedPath(abc.Traversable):
- """
- Given a series of Traversable objects, implement a merged
- version of the interface across all objects. Useful for
- namespace packages which may be multihomed at a single
- name.
- """
-
- def __init__(self, *paths):
- self._paths = list(map(pathlib.Path, remove_duplicates(paths)))
- if not self._paths:
- message = 'MultiplexedPath must contain at least one path'
- raise FileNotFoundError(message)
- if not all(path.is_dir() for path in self._paths):
- raise NotADirectoryError('MultiplexedPath only supports directories')
-
- def iterdir(self):
- visited = []
- for path in self._paths:
- for file in path.iterdir():
- if file.name in visited:
- continue
- visited.append(file.name)
- yield file
-
- def read_bytes(self):
- raise FileNotFoundError(f'{self} is not a file')
-
- def read_text(self, *args, **kwargs):
- raise FileNotFoundError(f'{self} is not a file')
-
- def is_dir(self):
- return True
-
- def is_file(self):
- return False
-
- def joinpath(self, child):
- # first try to find child in current paths
- for file in self.iterdir():
- if file.name == child:
- return file
- # if it does not exist, construct it with the first path
- return self._paths[0] / child
-
- __truediv__ = joinpath
-
- def open(self, *args, **kwargs):
- raise FileNotFoundError(f'{self} is not a file')
-
- @property
- def name(self):
- return self._paths[0].name
-
- def __repr__(self):
- paths = ', '.join(f"'{path}'" for path in self._paths)
- return f'MultiplexedPath({paths})'
-
-
-class NamespaceReader(abc.TraversableResources):
- def __init__(self, namespace_path):
- if 'NamespacePath' not in str(namespace_path):
- raise ValueError('Invalid path')
- self.path = MultiplexedPath(*list(namespace_path))
-
- def resource_path(self, resource):
- """
- Return the file system path to prevent
- `resources.path()` from creating a temporary
- copy.
- """
- return str(self.path.joinpath(resource))
-
- def files(self):
- return self.path
+__all__ = ['FileReader', 'ZipReader', 'MultiplexedPath', 'NamespaceReader']
diff --git a/common/py3-stdlib/importlib/resources.py b/common/py3-stdlib/importlib/resources.py
deleted file mode 100644
index 8a98663..0000000
--- a/common/py3-stdlib/importlib/resources.py
+++ /dev/null
@@ -1,185 +0,0 @@
-import os
-import io
-
-from . import _common
-from ._common import as_file, files
-from .abc import ResourceReader
-from contextlib import suppress
-from importlib.abc import ResourceLoader
-from importlib.machinery import ModuleSpec
-from io import BytesIO, TextIOWrapper
-from pathlib import Path
-from types import ModuleType
-from typing import ContextManager, Iterable, Union
-from typing import cast
-from typing.io import BinaryIO, TextIO
-from collections.abc import Sequence
-from functools import singledispatch
-
-
-__all__ = [
- 'Package',
- 'Resource',
- 'ResourceReader',
- 'as_file',
- 'contents',
- 'files',
- 'is_resource',
- 'open_binary',
- 'open_text',
- 'path',
- 'read_binary',
- 'read_text',
-]
-
-
-Package = Union[str, ModuleType]
-Resource = Union[str, os.PathLike]
-
-
-def open_binary(package: Package, resource: Resource) -> BinaryIO:
- """Return a file-like object opened for binary reading of the resource."""
- resource = _common.normalize_path(resource)
- package = _common.get_package(package)
- reader = _common.get_resource_reader(package)
- if reader is not None:
- return reader.open_resource(resource)
- spec = cast(ModuleSpec, package.__spec__)
- # Using pathlib doesn't work well here due to the lack of 'strict'
- # argument for pathlib.Path.resolve() prior to Python 3.6.
- if spec.submodule_search_locations is not None:
- paths = spec.submodule_search_locations
- elif spec.origin is not None:
- paths = [os.path.dirname(os.path.abspath(spec.origin))]
-
- for package_path in paths:
- full_path = os.path.join(package_path, resource)
- try:
- return open(full_path, mode='rb')
- except OSError:
- # Just assume the loader is a resource loader; all the relevant
- # importlib.machinery loaders are and an AttributeError for
- # get_data() will make it clear what is needed from the loader.
- loader = cast(ResourceLoader, spec.loader)
- data = None
- if hasattr(spec.loader, 'get_data'):
- with suppress(OSError):
- data = loader.get_data(full_path)
- if data is not None:
- return BytesIO(data)
-
- raise FileNotFoundError(f'{resource!r} resource not found in {spec.name!r}')
-
-
-def open_text(
- package: Package,
- resource: Resource,
- encoding: str = 'utf-8',
- errors: str = 'strict',
-) -> TextIO:
- """Return a file-like object opened for text reading of the resource."""
- return TextIOWrapper(
- open_binary(package, resource), encoding=encoding, errors=errors
- )
-
-
-def read_binary(package: Package, resource: Resource) -> bytes:
- """Return the binary contents of the resource."""
- with open_binary(package, resource) as fp:
- return fp.read()
-
-
-def read_text(
- package: Package,
- resource: Resource,
- encoding: str = 'utf-8',
- errors: str = 'strict',
-) -> str:
- """Return the decoded string of the resource.
-
- The decoding-related arguments have the same semantics as those of
- bytes.decode().
- """
- with open_text(package, resource, encoding, errors) as fp:
- return fp.read()
-
-
-def path(
- package: Package,
- resource: Resource,
-) -> 'ContextManager[Path]':
- """A context manager providing a file path object to the resource.
-
- If the resource does not already exist on its own on the file system,
- a temporary file will be created. If the file was created, the file
- will be deleted upon exiting the context manager (no exception is
- raised if the file was deleted prior to the context manager
- exiting).
- """
- reader = _common.get_resource_reader(_common.get_package(package))
- return (
- _path_from_reader(reader, _common.normalize_path(resource))
- if reader
- else _common.as_file(
- _common.files(package).joinpath(_common.normalize_path(resource))
- )
- )
-
-
-def _path_from_reader(reader, resource):
- return _path_from_resource_path(reader, resource) or _path_from_open_resource(
- reader, resource
- )
-
-
-def _path_from_resource_path(reader, resource):
- with suppress(FileNotFoundError):
- return Path(reader.resource_path(resource))
-
-
-def _path_from_open_resource(reader, resource):
- saved = io.BytesIO(reader.open_resource(resource).read())
- return _common._tempfile(saved.read, suffix=resource)
-
-
-def is_resource(package: Package, name: str) -> bool:
- """True if 'name' is a resource inside 'package'.
-
- Directories are *not* resources.
- """
- package = _common.get_package(package)
- _common.normalize_path(name)
- reader = _common.get_resource_reader(package)
- if reader is not None:
- return reader.is_resource(name)
- package_contents = set(contents(package))
- if name not in package_contents:
- return False
- return (_common.from_package(package) / name).is_file()
-
-
-def contents(package: Package) -> Iterable[str]:
- """Return an iterable of entries in 'package'.
-
- Note that not all entries are resources. Specifically, directories are
- not considered resources. Use `is_resource()` on each entry returned here
- to check if it is a resource or not.
- """
- package = _common.get_package(package)
- reader = _common.get_resource_reader(package)
- if reader is not None:
- return _ensure_sequence(reader.contents())
- transversable = _common.from_package(package)
- if transversable.is_dir():
- return list(item.name for item in transversable.iterdir())
- return []
-
-
-@singledispatch
-def _ensure_sequence(iterable):
- return list(iterable)
-
-
-@_ensure_sequence.register(Sequence)
-def _(iterable):
- return iterable
diff --git a/common/py3-stdlib/importlib/resources/__init__.py b/common/py3-stdlib/importlib/resources/__init__.py
new file mode 100644
index 0000000..34e3a99
--- /dev/null
+++ b/common/py3-stdlib/importlib/resources/__init__.py
@@ -0,0 +1,36 @@
+"""Read resources contained within a package."""
+
+from ._common import (
+ as_file,
+ files,
+ Package,
+)
+
+from ._legacy import (
+ contents,
+ open_binary,
+ read_binary,
+ open_text,
+ read_text,
+ is_resource,
+ path,
+ Resource,
+)
+
+from .abc import ResourceReader
+
+
+__all__ = [
+ 'Package',
+ 'Resource',
+ 'ResourceReader',
+ 'as_file',
+ 'contents',
+ 'files',
+ 'is_resource',
+ 'open_binary',
+ 'open_text',
+ 'path',
+ 'read_binary',
+ 'read_text',
+]
diff --git a/common/py3-stdlib/importlib/resources/_adapters.py b/common/py3-stdlib/importlib/resources/_adapters.py
new file mode 100644
index 0000000..ea363d8
--- /dev/null
+++ b/common/py3-stdlib/importlib/resources/_adapters.py
@@ -0,0 +1,170 @@
+from contextlib import suppress
+from io import TextIOWrapper
+
+from . import abc
+
+
+class SpecLoaderAdapter:
+ """
+ Adapt a package spec to adapt the underlying loader.
+ """
+
+ def __init__(self, spec, adapter=lambda spec: spec.loader):
+ self.spec = spec
+ self.loader = adapter(spec)
+
+ def __getattr__(self, name):
+ return getattr(self.spec, name)
+
+
+class TraversableResourcesLoader:
+ """
+ Adapt a loader to provide TraversableResources.
+ """
+
+ def __init__(self, spec):
+ self.spec = spec
+
+ def get_resource_reader(self, name):
+ return CompatibilityFiles(self.spec)._native()
+
+
+def _io_wrapper(file, mode='r', *args, **kwargs):
+ if mode == 'r':
+ return TextIOWrapper(file, *args, **kwargs)
+ elif mode == 'rb':
+ return file
+ raise ValueError(
+ "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode)
+ )
+
+
+class CompatibilityFiles:
+ """
+ Adapter for an existing or non-existent resource reader
+ to provide a compatibility .files().
+ """
+
+ class SpecPath(abc.Traversable):
+ """
+ Path tied to a module spec.
+ Can be read and exposes the resource reader children.
+ """
+
+ def __init__(self, spec, reader):
+ self._spec = spec
+ self._reader = reader
+
+ def iterdir(self):
+ if not self._reader:
+ return iter(())
+ return iter(
+ CompatibilityFiles.ChildPath(self._reader, path)
+ for path in self._reader.contents()
+ )
+
+ def is_file(self):
+ return False
+
+ is_dir = is_file
+
+ def joinpath(self, other):
+ if not self._reader:
+ return CompatibilityFiles.OrphanPath(other)
+ return CompatibilityFiles.ChildPath(self._reader, other)
+
+ @property
+ def name(self):
+ return self._spec.name
+
+ def open(self, mode='r', *args, **kwargs):
+ return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs)
+
+ class ChildPath(abc.Traversable):
+ """
+ Path tied to a resource reader child.
+ Can be read but doesn't expose any meaningful children.
+ """
+
+ def __init__(self, reader, name):
+ self._reader = reader
+ self._name = name
+
+ def iterdir(self):
+ return iter(())
+
+ def is_file(self):
+ return self._reader.is_resource(self.name)
+
+ def is_dir(self):
+ return not self.is_file()
+
+ def joinpath(self, other):
+ return CompatibilityFiles.OrphanPath(self.name, other)
+
+ @property
+ def name(self):
+ return self._name
+
+ def open(self, mode='r', *args, **kwargs):
+ return _io_wrapper(
+ self._reader.open_resource(self.name), mode, *args, **kwargs
+ )
+
+ class OrphanPath(abc.Traversable):
+ """
+ Orphan path, not tied to a module spec or resource reader.
+ Can't be read and doesn't expose any meaningful children.
+ """
+
+ def __init__(self, *path_parts):
+ if len(path_parts) < 1:
+ raise ValueError('Need at least one path part to construct a path')
+ self._path = path_parts
+
+ def iterdir(self):
+ return iter(())
+
+ def is_file(self):
+ return False
+
+ is_dir = is_file
+
+ def joinpath(self, other):
+ return CompatibilityFiles.OrphanPath(*self._path, other)
+
+ @property
+ def name(self):
+ return self._path[-1]
+
+ def open(self, mode='r', *args, **kwargs):
+ raise FileNotFoundError("Can't open orphan path")
+
+ def __init__(self, spec):
+ self.spec = spec
+
+ @property
+ def _reader(self):
+ with suppress(AttributeError):
+ return self.spec.loader.get_resource_reader(self.spec.name)
+
+ def _native(self):
+ """
+ Return the native reader if it supports files().
+ """
+ reader = self._reader
+ return reader if hasattr(reader, 'files') else self
+
+ def __getattr__(self, attr):
+ return getattr(self._reader, attr)
+
+ def files(self):
+ return CompatibilityFiles.SpecPath(self.spec, self._reader)
+
+
+def wrap_spec(package):
+ """
+ Construct a package spec with traversable compatibility
+ on the spec/loader/reader.
+ """
+ return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)
diff --git a/common/py3-stdlib/importlib/_common.py b/common/py3-stdlib/importlib/resources/_common.py
similarity index 85%
rename from common/py3-stdlib/importlib/_common.py
rename to common/py3-stdlib/importlib/resources/_common.py
index 84144c0..ca1fa8a 100644
--- a/common/py3-stdlib/importlib/_common.py
+++ b/common/py3-stdlib/importlib/resources/_common.py
@@ -6,7 +6,7 @@
import types
import importlib
-from typing import Union, Any, Optional
+from typing import Union, Optional
from .abc import ResourceReader, Traversable
from ._adapters import wrap_spec
@@ -22,19 +22,6 @@
return from_package(get_package(package))
-def normalize_path(path):
- # type: (Any) -> str
- """Normalize a path by ensuring it is a string.
-
- If the resulting string contains path separators, an exception is raised.
- """
- str_path = str(path)
- parent, file_name = os.path.split(str_path)
- if parent:
- raise ValueError(f'{path!r} must be only a file name')
- return file_name
-
-
def get_resource_reader(package):
# type: (types.ModuleType) -> Optional[ResourceReader]
"""
@@ -89,8 +76,10 @@
# properly.
fd, raw_path = tempfile.mkstemp(suffix=suffix)
try:
- os.write(fd, reader())
- os.close(fd)
+ try:
+ os.write(fd, reader())
+ finally:
+ os.close(fd)
del reader
yield pathlib.Path(raw_path)
finally:
diff --git a/common/py3-stdlib/importlib/resources/_itertools.py b/common/py3-stdlib/importlib/resources/_itertools.py
new file mode 100644
index 0000000..cce0558
--- /dev/null
+++ b/common/py3-stdlib/importlib/resources/_itertools.py
@@ -0,0 +1,35 @@
+from itertools import filterfalse
+
+from typing import (
+ Callable,
+ Iterable,
+ Iterator,
+ Optional,
+ Set,
+ TypeVar,
+ Union,
+)
+
+# Type and type variable definitions
+_T = TypeVar('_T')
+_U = TypeVar('_U')
+
+
+def unique_everseen(
+ iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None
+) -> Iterator[_T]:
+ "List unique elements, preserving order. Remember all elements ever seen."
+ # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+ # unique_everseen('ABBCcAD', str.lower) --> A B C D
+ seen: Set[Union[_T, _U]] = set()
+ seen_add = seen.add
+ if key is None:
+ for element in filterfalse(seen.__contains__, iterable):
+ seen_add(element)
+ yield element
+ else:
+ for element in iterable:
+ k = key(element)
+ if k not in seen:
+ seen_add(k)
+ yield element
diff --git a/common/py3-stdlib/importlib/resources/_legacy.py b/common/py3-stdlib/importlib/resources/_legacy.py
new file mode 100644
index 0000000..1d5d3f1
--- /dev/null
+++ b/common/py3-stdlib/importlib/resources/_legacy.py
@@ -0,0 +1,121 @@
+import functools
+import os
+import pathlib
+import types
+import warnings
+
+from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any
+
+from . import _common
+
+Package = Union[types.ModuleType, str]
+Resource = str
+
+
+def deprecated(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ warnings.warn(
+ f"{func.__name__} is deprecated. Use files() instead. "
+ "Refer to https://importlib-resources.readthedocs.io"
+ "/en/latest/using.html#migrating-from-legacy for migration advice.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return func(*args, **kwargs)
+
+ return wrapper
+
+
+def normalize_path(path):
+ # type: (Any) -> str
+ """Normalize a path by ensuring it is a string.
+
+ If the resulting string contains path separators, an exception is raised.
+ """
+ str_path = str(path)
+ parent, file_name = os.path.split(str_path)
+ if parent:
+ raise ValueError(f'{path!r} must be only a file name')
+ return file_name
+
+
+@deprecated
+def open_binary(package: Package, resource: Resource) -> BinaryIO:
+ """Return a file-like object opened for binary reading of the resource."""
+ return (_common.files(package) / normalize_path(resource)).open('rb')
+
+
+@deprecated
+def read_binary(package: Package, resource: Resource) -> bytes:
+ """Return the binary contents of the resource."""
+ return (_common.files(package) / normalize_path(resource)).read_bytes()
+
+
+@deprecated
+def open_text(
+ package: Package,
+ resource: Resource,
+ encoding: str = 'utf-8',
+ errors: str = 'strict',
+) -> TextIO:
+ """Return a file-like object opened for text reading of the resource."""
+ return (_common.files(package) / normalize_path(resource)).open(
+ 'r', encoding=encoding, errors=errors
+ )
+
+
+@deprecated
+def read_text(
+ package: Package,
+ resource: Resource,
+ encoding: str = 'utf-8',
+ errors: str = 'strict',
+) -> str:
+ """Return the decoded string of the resource.
+
+ The decoding-related arguments have the same semantics as those of
+ bytes.decode().
+ """
+ with open_text(package, resource, encoding, errors) as fp:
+ return fp.read()
+
+
+@deprecated
+def contents(package: Package) -> Iterable[str]:
+ """Return an iterable of entries in `package`.
+
+ Note that not all entries are resources. Specifically, directories are
+ not considered resources. Use `is_resource()` on each entry returned here
+ to check if it is a resource or not.
+ """
+ return [path.name for path in _common.files(package).iterdir()]
+
+
+@deprecated
+def is_resource(package: Package, name: str) -> bool:
+ """True if `name` is a resource inside `package`.
+
+ Directories are *not* resources.
+ """
+ resource = normalize_path(name)
+ return any(
+ traversable.name == resource and traversable.is_file()
+ for traversable in _common.files(package).iterdir()
+ )
+
+
+@deprecated
+def path(
+ package: Package,
+ resource: Resource,
+) -> ContextManager[pathlib.Path]:
+ """A context manager providing a file path object to the resource.
+
+ If the resource does not already exist on its own on the file system,
+ a temporary file will be created. If the file was created, the file
+ will be deleted upon exiting the context manager (no exception is
+ raised if the file was deleted prior to the context manager
+ exiting).
+ """
+ return _common.as_file(_common.files(package) / normalize_path(resource))
diff --git a/common/py3-stdlib/importlib/resources/abc.py b/common/py3-stdlib/importlib/resources/abc.py
new file mode 100644
index 0000000..0b7bfdc
--- /dev/null
+++ b/common/py3-stdlib/importlib/resources/abc.py
@@ -0,0 +1,151 @@
+import abc
+import io
+import os
+from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional
+from typing import runtime_checkable, Protocol
+from typing import Union
+
+
+StrPath = Union[str, os.PathLike[str]]
+
+__all__ = ["ResourceReader", "Traversable", "TraversableResources"]
+
+
+class ResourceReader(metaclass=abc.ABCMeta):
+ """Abstract base class for loaders to provide resource reading support."""
+
+ @abc.abstractmethod
+ def open_resource(self, resource: Text) -> BinaryIO:
+ """Return an opened, file-like object for binary reading.
+
+ The 'resource' argument is expected to represent only a file name.
+ If the resource cannot be found, FileNotFoundError is raised.
+ """
+ # This deliberately raises FileNotFoundError instead of
+ # NotImplementedError so that if this method is accidentally called,
+ # it'll still do the right thing.
+ raise FileNotFoundError
+
+ @abc.abstractmethod
+ def resource_path(self, resource: Text) -> Text:
+ """Return the file system path to the specified resource.
+
+ The 'resource' argument is expected to represent only a file name.
+ If the resource does not exist on the file system, raise
+ FileNotFoundError.
+ """
+ # This deliberately raises FileNotFoundError instead of
+ # NotImplementedError so that if this method is accidentally called,
+ # it'll still do the right thing.
+ raise FileNotFoundError
+
+ @abc.abstractmethod
+ def is_resource(self, path: Text) -> bool:
+ """Return True if the named 'path' is a resource.
+
+ Files are resources, directories are not.
+ """
+ raise FileNotFoundError
+
+ @abc.abstractmethod
+ def contents(self) -> Iterable[str]:
+ """Return an iterable of entries in `package`."""
+ raise FileNotFoundError
+
+
+@runtime_checkable
+class Traversable(Protocol):
+ """
+ An object with a subset of pathlib.Path methods suitable for
+ traversing directories and opening files.
+
+ Any exceptions that occur when accessing the backing resource
+ may propagate unaltered.
+ """
+
+ @abc.abstractmethod
+ def iterdir(self) -> Iterator["Traversable"]:
+ """
+ Yield Traversable objects in self
+ """
+
+ def read_bytes(self) -> bytes:
+ """
+ Read contents of self as bytes
+ """
+ with self.open('rb') as strm:
+ return strm.read()
+
+ def read_text(self, encoding: Optional[str] = None) -> str:
+ """
+ Read contents of self as text
+ """
+ with self.open(encoding=encoding) as strm:
+ return strm.read()
+
+ @abc.abstractmethod
+ def is_dir(self) -> bool:
+ """
+ Return True if self is a directory
+ """
+
+ @abc.abstractmethod
+ def is_file(self) -> bool:
+ """
+ Return True if self is a file
+ """
+
+ @abc.abstractmethod
+ def joinpath(self, *descendants: StrPath) -> "Traversable":
+ """
+ Return Traversable resolved with any descendants applied.
+
+ Each descendant should be a path segment relative to self
+ and each may contain multiple levels separated by
+ ``posixpath.sep`` (``/``).
+ """
+
+ def __truediv__(self, child: StrPath) -> "Traversable":
+ """
+ Return Traversable child in self
+ """
+ return self.joinpath(child)
+
+ @abc.abstractmethod
+ def open(self, mode='r', *args, **kwargs):
+ """
+ mode may be 'r' or 'rb' to open as text or binary. Return a handle
+ suitable for reading (same as pathlib.Path.open).
+
+ When opening as text, accepts encoding parameters such as those
+ accepted by io.TextIOWrapper.
+ """
+
+ @abc.abstractproperty
+ def name(self) -> str:
+ """
+ The base name of this object without any parent references.
+ """
+
+
+class TraversableResources(ResourceReader):
+ """
+ The required interface for providing traversable
+ resources.
+ """
+
+ @abc.abstractmethod
+ def files(self) -> "Traversable":
+ """Return a Traversable object for the loaded package."""
+
+ def open_resource(self, resource: StrPath) -> io.BufferedReader:
+ return self.files().joinpath(resource).open('rb')
+
+ def resource_path(self, resource: Any) -> NoReturn:
+ raise FileNotFoundError(resource)
+
+ def is_resource(self, path: StrPath) -> bool:
+ return self.files().joinpath(path).is_file()
+
+ def contents(self) -> Iterator[str]:
+ return (item.name for item in self.files().iterdir())
diff --git a/common/py3-stdlib/importlib/resources/readers.py b/common/py3-stdlib/importlib/resources/readers.py
new file mode 100644
index 0000000..b470a20
--- /dev/null
+++ b/common/py3-stdlib/importlib/resources/readers.py
@@ -0,0 +1,122 @@
+import collections
+import operator
+import pathlib
+import zipfile
+
+from . import abc
+
+from ._itertools import unique_everseen
+
+
+def remove_duplicates(items):
+ return iter(collections.OrderedDict.fromkeys(items))
+
+
+class FileReader(abc.TraversableResources):
+ def __init__(self, loader):
+ self.path = pathlib.Path(loader.path).parent
+
+ def resource_path(self, resource):
+ """
+ Return the file system path to prevent
+ `resources.path()` from creating a temporary
+ copy.
+ """
+ return str(self.path.joinpath(resource))
+
+ def files(self):
+ return self.path
+
+
+class ZipReader(abc.TraversableResources):
+ def __init__(self, loader, module):
+ _, _, name = module.rpartition('.')
+ self.prefix = loader.prefix.replace('\\', '/') + name + '/'
+ self.archive = loader.archive
+
+ def open_resource(self, resource):
+ try:
+ return super().open_resource(resource)
+ except KeyError as exc:
+ raise FileNotFoundError(exc.args[0])
+
+ def is_resource(self, path):
+ # workaround for `zipfile.Path.is_file` returning true
+ # for non-existent paths.
+ target = self.files().joinpath(path)
+ return target.is_file() and target.exists()
+
+ def files(self):
+ return zipfile.Path(self.archive, self.prefix)
+
+
+class MultiplexedPath(abc.Traversable):
+ """
+ Given a series of Traversable objects, implement a merged
+ version of the interface across all objects. Useful for
+ namespace packages which may be multihomed at a single
+ name.
+ """
+
+ def __init__(self, *paths):
+ self._paths = list(map(pathlib.Path, remove_duplicates(paths)))
+ if not self._paths:
+ message = 'MultiplexedPath must contain at least one path'
+ raise FileNotFoundError(message)
+ if not all(path.is_dir() for path in self._paths):
+ raise NotADirectoryError('MultiplexedPath only supports directories')
+
+ def iterdir(self):
+ files = (file for path in self._paths for file in path.iterdir())
+ return unique_everseen(files, key=operator.attrgetter('name'))
+
+ def read_bytes(self):
+ raise FileNotFoundError(f'{self} is not a file')
+
+ def read_text(self, *args, **kwargs):
+ raise FileNotFoundError(f'{self} is not a file')
+
+ def is_dir(self):
+ return True
+
+ def is_file(self):
+ return False
+
+ def joinpath(self, child):
+ # first try to find child in current paths
+ for file in self.iterdir():
+ if file.name == child:
+ return file
+ # if it does not exist, construct it with the first path
+ return self._paths[0] / child
+
+ __truediv__ = joinpath
+
+ def open(self, *args, **kwargs):
+ raise FileNotFoundError(f'{self} is not a file')
+
+ @property
+ def name(self):
+ return self._paths[0].name
+
+ def __repr__(self):
+ paths = ', '.join(f"'{path}'" for path in self._paths)
+ return f'MultiplexedPath({paths})'
+
+
+class NamespaceReader(abc.TraversableResources):
+ def __init__(self, namespace_path):
+ if 'NamespacePath' not in str(namespace_path):
+ raise ValueError('Invalid path')
+ self.path = MultiplexedPath(*list(namespace_path))
+
+ def resource_path(self, resource):
+ """
+ Return the file system path to prevent
+ `resources.path()` from creating a temporary
+ copy.
+ """
+ return str(self.path.joinpath(resource))
+
+ def files(self):
+ return self.path
diff --git a/common/py3-stdlib/importlib/resources/simple.py b/common/py3-stdlib/importlib/resources/simple.py
new file mode 100644
index 0000000..d0fbf23
--- /dev/null
+++ b/common/py3-stdlib/importlib/resources/simple.py
@@ -0,0 +1,125 @@
+"""
+Interface adapters for low-level readers.
+"""
+
+import abc
+import io
+import itertools
+from typing import BinaryIO, List
+
+from .abc import Traversable, TraversableResources
+
+
+class SimpleReader(abc.ABC):
+ """
+ The minimum, low-level interface required from a resource
+ provider.
+ """
+
+ @abc.abstractproperty
+ def package(self):
+ # type: () -> str
+ """
+ The name of the package for which this reader loads resources.
+ """
+
+ @abc.abstractmethod
+ def children(self):
+ # type: () -> List['SimpleReader']
+ """
+ Obtain an iterable of SimpleReader for available
+ child containers (e.g. directories).
+ """
+
+ @abc.abstractmethod
+ def resources(self):
+ # type: () -> List[str]
+ """
+ Obtain available named resources for this virtual package.
+ """
+
+ @abc.abstractmethod
+ def open_binary(self, resource):
+ # type: (str) -> BinaryIO
+ """
+ Obtain a File-like for a named resource.
+ """
+
+ @property
+ def name(self):
+ return self.package.split('.')[-1]
+
+
+class ResourceHandle(Traversable):
+ """
+ Handle to a named resource in a ResourceReader.
+ """
+
+ def __init__(self, parent, name):
+ # type: (ResourceContainer, str) -> None
+ self.parent = parent
+ self.name = name # type: ignore
+
+ def is_file(self):
+ return True
+
+ def is_dir(self):
+ return False
+
+ def open(self, mode='r', *args, **kwargs):
+ stream = self.parent.reader.open_binary(self.name)
+ if 'b' not in mode:
+ stream = io.TextIOWrapper(*args, **kwargs)
+ return stream
+
+ def joinpath(self, name):
+ raise RuntimeError("Cannot traverse into a resource")
+
+
+class ResourceContainer(Traversable):
+ """
+ Traversable container for a package's resources via its reader.
+ """
+
+ def __init__(self, reader):
+ # type: (SimpleReader) -> None
+ self.reader = reader
+
+ def is_dir(self):
+ return True
+
+ def is_file(self):
+ return False
+
+ def iterdir(self):
+ files = (ResourceHandle(self, name) for name in self.reader.resources)
+ dirs = map(ResourceContainer, self.reader.children())
+ return itertools.chain(files, dirs)
+
+ def open(self, *args, **kwargs):
+ raise IsADirectoryError()
+
+ @staticmethod
+ def _flatten(compound_names):
+ for name in compound_names:
+ yield from name.split('/')
+
+ def joinpath(self, *descendants):
+ if not descendants:
+ return self
+ names = self._flatten(descendants)
+ target = next(names)
+ return next(
+ traversable for traversable in self.iterdir() if traversable.name == target
+ ).joinpath(*names)
+
+
+class TraversableReader(TraversableResources, SimpleReader):
+ """
+ A TraversableResources based on SimpleReader. Resource providers
+ may derive from this class to provide the TraversableResources
+ interface by supplying the SimpleReader interface.
+ """
+
+ def files(self):
+ return ResourceContainer(self)
diff --git a/common/py3-stdlib/importlib/simple.py b/common/py3-stdlib/importlib/simple.py
new file mode 100644
index 0000000..845bb90
--- /dev/null
+++ b/common/py3-stdlib/importlib/simple.py
@@ -0,0 +1,14 @@
+"""
+Compatibility shim for .resources.simple as found on Python 3.10.
+
+Consumers that can rely on Python 3.11 should use the other
+module directly.
+"""
+
+from .resources.simple import (
+ SimpleReader, ResourceHandle, ResourceContainer, TraversableReader,
+)
+
+__all__ = [
+ 'SimpleReader', 'ResourceHandle', 'ResourceContainer', 'TraversableReader',
+]
diff --git a/common/py3-stdlib/inspect.py b/common/py3-stdlib/inspect.py
index 2999a60..655b04b 100644
--- a/common/py3-stdlib/inspect.py
+++ b/common/py3-stdlib/inspect.py
@@ -33,6 +33,106 @@
__author__ = ('Ka-Ping Yee <ping@lfw.org>',
'Yury Selivanov <yselivanov@sprymix.com>')
+__all__ = [
+ "ArgInfo",
+ "Arguments",
+ "Attribute",
+ "BlockFinder",
+ "BoundArguments",
+ "CORO_CLOSED",
+ "CORO_CREATED",
+ "CORO_RUNNING",
+ "CORO_SUSPENDED",
+ "CO_ASYNC_GENERATOR",
+ "CO_COROUTINE",
+ "CO_GENERATOR",
+ "CO_ITERABLE_COROUTINE",
+ "CO_NESTED",
+ "CO_NEWLOCALS",
+ "CO_NOFREE",
+ "CO_OPTIMIZED",
+ "CO_VARARGS",
+ "CO_VARKEYWORDS",
+ "ClassFoundException",
+ "ClosureVars",
+ "EndOfBlock",
+ "FrameInfo",
+ "FullArgSpec",
+ "GEN_CLOSED",
+ "GEN_CREATED",
+ "GEN_RUNNING",
+ "GEN_SUSPENDED",
+ "Parameter",
+ "Signature",
+ "TPFLAGS_IS_ABSTRACT",
+ "Traceback",
+ "classify_class_attrs",
+ "cleandoc",
+ "currentframe",
+ "findsource",
+ "formatannotation",
+ "formatannotationrelativeto",
+ "formatargvalues",
+ "get_annotations",
+ "getabsfile",
+ "getargs",
+ "getargvalues",
+ "getattr_static",
+ "getblock",
+ "getcallargs",
+ "getclasstree",
+ "getclosurevars",
+ "getcomments",
+ "getcoroutinelocals",
+ "getcoroutinestate",
+ "getdoc",
+ "getfile",
+ "getframeinfo",
+ "getfullargspec",
+ "getgeneratorlocals",
+ "getgeneratorstate",
+ "getinnerframes",
+ "getlineno",
+ "getmembers",
+ "getmembers_static",
+ "getmodule",
+ "getmodulename",
+ "getmro",
+ "getouterframes",
+ "getsource",
+ "getsourcefile",
+ "getsourcelines",
+ "indentsize",
+ "isabstract",
+ "isasyncgen",
+ "isasyncgenfunction",
+ "isawaitable",
+ "isbuiltin",
+ "isclass",
+ "iscode",
+ "iscoroutine",
+ "iscoroutinefunction",
+ "isdatadescriptor",
+ "isframe",
+ "isfunction",
+ "isgenerator",
+ "isgeneratorfunction",
+ "isgetsetdescriptor",
+ "ismemberdescriptor",
+ "ismethod",
+ "ismethoddescriptor",
+ "ismethodwrapper",
+ "ismodule",
+ "isroutine",
+ "istraceback",
+ "signature",
+ "stack",
+ "trace",
+ "unwrap",
+ "walktree",
+]
+
+
import abc
import ast
import dis
@@ -47,9 +147,9 @@
import tokenize
import token
import types
-import warnings
import functools
import builtins
+from keyword import iskeyword
from operator import attrgetter
from collections import namedtuple, OrderedDict
@@ -58,6 +158,7 @@
mod_dict = globals()
for k, v in dis.COMPILER_FLAG_NAMES.items():
mod_dict["CO_" + v] = k
+del k, v, mod_dict
# See Include/object.h
TPFLAGS_IS_ABSTRACT = 1 << 20
@@ -410,12 +511,17 @@
__self__ instance to which a method is bound, or None"""
return isinstance(object, types.BuiltinFunctionType)
+def ismethodwrapper(object):
+ """Return true if the object is a method wrapper."""
+ return isinstance(object, types.MethodWrapperType)
+
def isroutine(object):
"""Return true if the object is any kind of function or method."""
return (isbuiltin(object)
or isfunction(object)
or ismethod(object)
- or ismethoddescriptor(object))
+ or ismethoddescriptor(object)
+ or ismethodwrapper(object))
def isabstract(object):
"""Return true if the object is an abstract base class (ABC)."""
@@ -441,32 +547,30 @@
return True
return False
-def getmembers(object, predicate=None):
- """Return all members of an object as (name, value) pairs sorted by name.
- Optionally, only return members that satisfy a given predicate."""
- if isclass(object):
- mro = (object,) + getmro(object)
- else:
- mro = ()
+def _getmembers(object, predicate, getter):
results = []
processed = set()
names = dir(object)
- # :dd any DynamicClassAttributes to the list of names if object is a class;
- # this may result in duplicate entries if, for example, a virtual
- # attribute with the same name as a DynamicClassAttribute exists
- try:
- for base in object.__bases__:
- for k, v in base.__dict__.items():
- if isinstance(v, types.DynamicClassAttribute):
- names.append(k)
- except AttributeError:
- pass
+ if isclass(object):
+ mro = (object,) + getmro(object)
+ # add any DynamicClassAttributes to the list of names if object is a class;
+ # this may result in duplicate entries if, for example, a virtual
+ # attribute with the same name as a DynamicClassAttribute exists
+ try:
+ for base in object.__bases__:
+ for k, v in base.__dict__.items():
+ if isinstance(v, types.DynamicClassAttribute):
+ names.append(k)
+ except AttributeError:
+ pass
+ else:
+ mro = ()
for key in names:
# First try to get the value via getattr. Some descriptors don't
# like calling their __get__ (see bug #1785), so fall back to
# looking in the __dict__.
try:
- value = getattr(object, key)
+ value = getter(object, key)
# handle the duplicate key
if key in processed:
raise AttributeError
@@ -485,6 +589,25 @@
results.sort(key=lambda pair: pair[0])
return results
+def getmembers(object, predicate=None):
+ """Return all members of an object as (name, value) pairs sorted by name.
+ Optionally, only return members that satisfy a given predicate."""
+ return _getmembers(object, predicate, getattr)
+
+def getmembers_static(object, predicate=None):
+ """Return all members of an object as (name, value) pairs sorted by name
+ without triggering dynamic lookup via the descriptor protocol,
+ __getattr__ or __getattribute__. Optionally, only return members that
+ satisfy a given predicate.
+
+ Note: this function may not be able to retrieve all members
+ that getmembers can fetch (like dynamically created attributes)
+ and may find members that getmembers can't (like descriptors
+ that raise AttributeError). It can also return descriptor objects
+ instead of instance members in some cases.
+ """
+ return _getmembers(object, predicate, getattr_static)
+
Attribute = namedtuple('Attribute', 'name kind defining_class object')
def classify_class_attrs(cls):
@@ -1206,37 +1329,6 @@
varkw = co.co_varnames[nargs]
return Arguments(args + kwonlyargs, varargs, varkw)
-ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults')
-
-def getargspec(func):
- """Get the names and default values of a function's parameters.
-
- A tuple of four things is returned: (args, varargs, keywords, defaults).
- 'args' is a list of the argument names, including keyword-only argument names.
- 'varargs' and 'keywords' are the names of the * and ** parameters or None.
- 'defaults' is an n-tuple of the default values of the last n parameters.
-
- This function is deprecated, as it does not support annotations or
- keyword-only parameters and will raise ValueError if either is present
- on the supplied callable.
-
- For a more structured introspection API, use inspect.signature() instead.
-
- Alternatively, use getfullargspec() for an API with a similar namedtuple
- based interface, but full support for annotations and keyword-only
- parameters.
-
- Deprecated since Python 3.5, use `inspect.getfullargspec()`.
- """
- warnings.warn("inspect.getargspec() is deprecated since Python 3.0, "
- "use inspect.signature() or inspect.getfullargspec()",
- DeprecationWarning, stacklevel=2)
- args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \
- getfullargspec(func)
- if kwonlyargs or ann:
- raise ValueError("Function has keyword-only parameters or annotations"
- ", use inspect.signature() API which can support them")
- return ArgSpec(args, varargs, varkw, defaults)
FullArgSpec = namedtuple('FullArgSpec',
'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations')
@@ -1366,63 +1458,6 @@
return formatannotation(annotation, module)
return _formatannotation
-def formatargspec(args, varargs=None, varkw=None, defaults=None,
- kwonlyargs=(), kwonlydefaults={}, annotations={},
- formatarg=str,
- formatvarargs=lambda name: '*' + name,
- formatvarkw=lambda name: '**' + name,
- formatvalue=lambda value: '=' + repr(value),
- formatreturns=lambda text: ' -> ' + text,
- formatannotation=formatannotation):
- """Format an argument spec from the values returned by getfullargspec.
-
- The first seven arguments are (args, varargs, varkw, defaults,
- kwonlyargs, kwonlydefaults, annotations). The other five arguments
- are the corresponding optional formatting functions that are called to
- turn names and values into strings. The last argument is an optional
- function to format the sequence of arguments.
-
- Deprecated since Python 3.5: use the `signature` function and `Signature`
- objects.
- """
-
- from warnings import warn
-
- warn("`formatargspec` is deprecated since Python 3.5. Use `signature` and "
- "the `Signature` object directly",
- DeprecationWarning,
- stacklevel=2)
-
- def formatargandannotation(arg):
- result = formatarg(arg)
- if arg in annotations:
- result += ': ' + formatannotation(annotations[arg])
- return result
- specs = []
- if defaults:
- firstdefault = len(args) - len(defaults)
- for i, arg in enumerate(args):
- spec = formatargandannotation(arg)
- if defaults and i >= firstdefault:
- spec = spec + formatvalue(defaults[i - firstdefault])
- specs.append(spec)
- if varargs is not None:
- specs.append(formatvarargs(formatargandannotation(varargs)))
- else:
- if kwonlyargs:
- specs.append('*')
- if kwonlyargs:
- for kwonlyarg in kwonlyargs:
- spec = formatargandannotation(kwonlyarg)
- if kwonlydefaults and kwonlyarg in kwonlydefaults:
- spec += formatvalue(kwonlydefaults[kwonlyarg])
- specs.append(spec)
- if varkw is not None:
- specs.append(formatvarkw(formatargandannotation(varkw)))
- result = '(' + ', '.join(specs) + ')'
- if 'return' in annotations:
- result += formatreturns(formatannotation(annotations['return']))
- return result
def formatargvalues(args, varargs, varkw, locals,
formatarg=str,
@@ -1599,7 +1634,30 @@
# -------------------------------------------------- stack frame extraction
-Traceback = namedtuple('Traceback', 'filename lineno function code_context index')
+_Traceback = namedtuple('_Traceback', 'filename lineno function code_context index')
+
+class Traceback(_Traceback):
+ def __new__(cls, filename, lineno, function, code_context, index, *, positions=None):
+ instance = super().__new__(cls, filename, lineno, function, code_context, index)
+ instance.positions = positions
+ return instance
+
+ def __repr__(self):
+ return ('Traceback(filename={!r}, lineno={!r}, function={!r}, '
+ 'code_context={!r}, index={!r}, positions={!r})'.format(
+ self.filename, self.lineno, self.function, self.code_context,
+ self.index, self.positions))
+
+def _get_code_position_from_tb(tb):
+ code, instruction_index = tb.tb_frame.f_code, tb.tb_lasti
+ return _get_code_position(code, instruction_index)
+
+def _get_code_position(code, instruction_index):
+ if instruction_index < 0:
+ return (None, None, None, None)
+ positions_gen = code.co_positions()
+ # The nth entry in code.co_positions() corresponds to instruction (2*n)th since Python 3.10+
+ return next(itertools.islice(positions_gen, instruction_index // 2, None))
def getframeinfo(frame, context=1):
"""Get information about a frame or traceback object.
@@ -1610,10 +1668,20 @@
The optional second argument specifies the number of lines of context
to return, which are centered around the current line."""
if istraceback(frame):
+ positions = _get_code_position_from_tb(frame)
lineno = frame.tb_lineno
frame = frame.tb_frame
else:
lineno = frame.f_lineno
+ positions = _get_code_position(frame.f_code, frame.f_lasti)
+
+ if positions[0] is None:
+ frame, *positions = (frame, lineno, *positions[1:])
+ else:
+ frame, *positions = (frame, *positions)
+
+ lineno = positions[0]
+
if not isframe(frame):
raise TypeError('{!r} is not a frame or traceback object'.format(frame))
@@ -1631,14 +1699,26 @@
else:
lines = index = None
- return Traceback(filename, lineno, frame.f_code.co_name, lines, index)
+ return Traceback(filename, lineno, frame.f_code.co_name, lines,
+ index, positions=dis.Positions(*positions))
def getlineno(frame):
"""Get the line number from a frame object, allowing for optimization."""
# FrameType.f_lineno is now a descriptor that grovels co_lnotab
return frame.f_lineno
-FrameInfo = namedtuple('FrameInfo', ('frame',) + Traceback._fields)
+_FrameInfo = namedtuple('_FrameInfo', ('frame',) + Traceback._fields)
+class FrameInfo(_FrameInfo):
+ def __new__(cls, frame, filename, lineno, function, code_context, index, *, positions=None):
+ instance = super().__new__(cls, frame, filename, lineno, function, code_context, index)
+ instance.positions = positions
+ return instance
+
+ def __repr__(self):
+ return ('FrameInfo(frame={!r}, filename={!r}, lineno={!r}, function={!r}, '
+ 'code_context={!r}, index={!r}, positions={!r})'.format(
+ self.frame, self.filename, self.lineno, self.function,
+ self.code_context, self.index, self.positions))
def getouterframes(frame, context=1):
"""Get a list of records for a frame and all higher (calling) frames.
@@ -1647,8 +1727,9 @@
name, a list of lines of context, and index within the context."""
framelist = []
while frame:
- frameinfo = (frame,) + getframeinfo(frame, context)
- framelist.append(FrameInfo(*frameinfo))
+ traceback_info = getframeinfo(frame, context)
+ frameinfo = (frame,) + traceback_info
+ framelist.append(FrameInfo(*frameinfo, positions=traceback_info.positions))
frame = frame.f_back
return framelist
@@ -1659,8 +1740,9 @@
name, a list of lines of context, and index within the context."""
framelist = []
while tb:
- frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)
- framelist.append(FrameInfo(*frameinfo))
+ traceback_info = getframeinfo(tb, context)
+ frameinfo = (tb.tb_frame,) + traceback_info
+ framelist.append(FrameInfo(*frameinfo, positions=traceback_info.positions))
tb = tb.tb_next
return framelist
@@ -1747,8 +1829,10 @@
klass_result = _check_class(klass, attr)
if instance_result is not _sentinel and klass_result is not _sentinel:
- if (_check_class(type(klass_result), '__get__') is not _sentinel and
- _check_class(type(klass_result), '__set__') is not _sentinel):
+ if _check_class(type(klass_result), "__get__") is not _sentinel and (
+ _check_class(type(klass_result), "__set__") is not _sentinel
+ or _check_class(type(klass_result), "__delete__") is not _sentinel
+ ):
return klass_result
if instance_result is not _sentinel:
@@ -1787,11 +1871,11 @@
"""
if generator.gi_running:
return GEN_RUNNING
+ if generator.gi_suspended:
+ return GEN_SUSPENDED
if generator.gi_frame is None:
return GEN_CLOSED
- if generator.gi_frame.f_lasti == -1:
- return GEN_CREATED
- return GEN_SUSPENDED
+ return GEN_CREATED
def getgeneratorlocals(generator):
@@ -1829,11 +1913,11 @@
"""
if coroutine.cr_running:
return CORO_RUNNING
+ if coroutine.cr_suspended:
+ return CORO_SUSPENDED
if coroutine.cr_frame is None:
return CORO_CLOSED
- if coroutine.cr_frame.f_lasti == -1:
- return CORO_CREATED
- return CORO_SUSPENDED
+ return CORO_CREATED
def getcoroutinelocals(coroutine):
@@ -1854,13 +1938,9 @@
###############################################################################
-_WrapperDescriptor = type(type.__call__)
-_MethodWrapper = type(all.__call__)
-_ClassMethodWrapper = type(int.__dict__['from_bytes'])
-
-_NonUserDefinedCallables = (_WrapperDescriptor,
- _MethodWrapper,
- _ClassMethodWrapper,
+_NonUserDefinedCallables = (types.WrapperDescriptorType,
+ types.MethodWrapperType,
+ types.ClassMethodDescriptorType,
types.BuiltinFunctionType)
@@ -2019,29 +2099,6 @@
(isinstance(annotations, (dict)) or annotations is None) )
-def _signature_get_bound_param(spec):
- """ Private helper to get first parameter name from a
- __text_signature__ of a builtin method, which should
- be in the following format: '($param1, ...)'.
- Assumptions are that the first argument won't have
- a default value or an annotation.
- """
-
- assert spec.startswith('($')
-
- pos = spec.find(',')
- if pos == -1:
- pos = spec.find(')')
-
- cpos = spec.find(':')
- assert cpos == -1 or cpos > pos
-
- cpos = spec.find('=')
- assert cpos == -1 or cpos > pos
-
- return spec[2:pos]
-
-
def _signature_strip_non_python_syntax(signature):
"""
Private helper function. Takes a signature in Argument Clinic's
@@ -2118,10 +2175,6 @@
"""Private helper to parse content of '__text_signature__'
and return a Signature based on it.
"""
- # Lazy import ast because it's relatively heavy and
- # it's not used for other than this function.
- import ast
-
Parameter = cls._parameter_cls
clean_signature, self_parameter, last_positional_only = \
@@ -2485,17 +2538,18 @@
factory_method = None
new = _signature_get_user_defined_method(obj, '__new__')
init = _signature_get_user_defined_method(obj, '__init__')
- # Now we check if the 'obj' class has an own '__new__' method
- if '__new__' in obj.__dict__:
- factory_method = new
- # or an own '__init__' method
- elif '__init__' in obj.__dict__:
- factory_method = init
- # If not, we take inherited '__new__' or '__init__', if present
- elif new is not None:
- factory_method = new
- elif init is not None:
- factory_method = init
+
+ # Go through the MRO and see if any class has user-defined
+ # pure Python __new__ or __init__ method
+ for base in obj.__mro__:
+ # Now we check if the 'obj' class has an own '__new__' method
+ if new is not None and '__new__' in base.__dict__:
+ factory_method = new
+ break
+ # or an own '__init__' method
+ elif init is not None and '__init__' in base.__dict__:
+ factory_method = init
+ break
if factory_method is not None:
sig = _get_signature_of(factory_method)
@@ -2539,7 +2593,7 @@
elif not isinstance(obj, _NonUserDefinedCallables):
# An object with __call__
# We also check that the 'obj' is not an instance of
- # _WrapperDescriptor or _MethodWrapper to avoid
+ # types.WrapperDescriptorType or types.MethodWrapperType to avoid
# infinite recursion (and even potential segfault)
call = _signature_get_user_defined_method(type(obj), '__call__')
if call is not None:
@@ -2574,18 +2628,21 @@
class _ParameterKind(enum.IntEnum):
- POSITIONAL_ONLY = 0
- POSITIONAL_OR_KEYWORD = 1
- VAR_POSITIONAL = 2
- KEYWORD_ONLY = 3
- VAR_KEYWORD = 4
+ POSITIONAL_ONLY = 'positional-only'
+ POSITIONAL_OR_KEYWORD = 'positional or keyword'
+ VAR_POSITIONAL = 'variadic positional'
+ KEYWORD_ONLY = 'keyword-only'
+ VAR_KEYWORD = 'variadic keyword'
+
+ def __new__(cls, description):
+ value = len(cls.__members__)
+ member = int.__new__(cls, value)
+ member._value_ = value
+ member.description = description
+ return member
def __str__(self):
- return self._name_
-
- @property
- def description(self):
- return _PARAM_NAME_MAPPING[self]
+ return self.name
_POSITIONAL_ONLY = _ParameterKind.POSITIONAL_ONLY
_POSITIONAL_OR_KEYWORD = _ParameterKind.POSITIONAL_OR_KEYWORD
@@ -2593,14 +2650,6 @@
_KEYWORD_ONLY = _ParameterKind.KEYWORD_ONLY
_VAR_KEYWORD = _ParameterKind.VAR_KEYWORD
-_PARAM_NAME_MAPPING = {
- _POSITIONAL_ONLY: 'positional-only',
- _POSITIONAL_OR_KEYWORD: 'positional or keyword',
- _VAR_POSITIONAL: 'variadic positional',
- _KEYWORD_ONLY: 'keyword-only',
- _VAR_KEYWORD: 'variadic keyword'
-}
-
class Parameter:
"""Represents a parameter in a function signature.
@@ -2669,7 +2718,10 @@
self._kind = _POSITIONAL_ONLY
name = 'implicit{}'.format(name[1:])
- if not name.isidentifier():
+ # It's possible for C functions to have a positional-only parameter
+ # where the name is a keyword, so for compatibility we'll allow it.
+ is_keyword = iskeyword(name) and self._kind is not _POSITIONAL_ONLY
+ if is_keyword or not name.isidentifier():
raise ValueError('{!r} is not a valid parameter name'.format(name))
self._name = name
@@ -2929,7 +2981,7 @@
if __validate_parameters__:
params = OrderedDict()
top_kind = _POSITIONAL_ONLY
- kind_defaults = False
+ seen_default = False
for param in parameters:
kind = param.kind
@@ -2944,21 +2996,19 @@
kind.description)
raise ValueError(msg)
elif kind > top_kind:
- kind_defaults = False
top_kind = kind
if kind in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD):
if param.default is _empty:
- if kind_defaults:
+ if seen_default:
# No default for this parameter, but the
- # previous parameter of the same kind had
- # a default
+ # previous parameter of had a default
msg = 'non-default argument follows default ' \
'argument'
raise ValueError(msg)
else:
# There is a default for this parameter.
- kind_defaults = True
+ seen_default = True
if name in params:
msg = 'duplicate parameter name: {!r}'.format(name)
@@ -2972,30 +3022,6 @@
self._return_annotation = return_annotation
@classmethod
- def from_function(cls, func):
- """Constructs Signature for the given python function.
-
- Deprecated since Python 3.5, use `Signature.from_callable()`.
- """
-
- warnings.warn("inspect.Signature.from_function() is deprecated since "
- "Python 3.5, use Signature.from_callable()",
- DeprecationWarning, stacklevel=2)
- return _signature_from_function(cls, func)
-
- @classmethod
- def from_builtin(cls, func):
- """Constructs Signature for the given builtin function.
-
- Deprecated since Python 3.5, use `Signature.from_callable()`.
- """
-
- warnings.warn("inspect.Signature.from_builtin() is deprecated since "
- "Python 3.5, use Signature.from_callable()",
- DeprecationWarning, stacklevel=2)
- return _signature_from_builtin(cls, func)
-
- @classmethod
def from_callable(cls, obj, *,
follow_wrapped=True, globals=None, locals=None, eval_str=False):
"""Constructs Signature for the given callable object."""
diff --git a/common/py3-stdlib/io.py b/common/py3-stdlib/io.py
index 2a6140c..a418649 100644
--- a/common/py3-stdlib/io.py
+++ b/common/py3-stdlib/io.py
@@ -70,7 +70,7 @@
global OpenWrapper
OpenWrapper = open
return OpenWrapper
- raise AttributeError(name)
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
# Pretend this exception was created here.
diff --git a/common/py3-stdlib/ipaddress.py b/common/py3-stdlib/ipaddress.py
index 756f1bc..1cb71d8 100644
--- a/common/py3-stdlib/ipaddress.py
+++ b/common/py3-stdlib/ipaddress.py
@@ -132,7 +132,7 @@
"""
try:
- return address.to_bytes(4, 'big')
+ return address.to_bytes(4) # big endian
except OverflowError:
raise ValueError("Address negative or too large for IPv4")
@@ -148,7 +148,7 @@
"""
try:
- return address.to_bytes(16, 'big')
+ return address.to_bytes(16) # big endian
except OverflowError:
raise ValueError("Address negative or too large for IPv6")
@@ -1077,15 +1077,16 @@
@property
def is_private(self):
- """Test if this address is allocated for private networks.
+ """Test if this network belongs to a private range.
Returns:
- A boolean, True if the address is reserved per
+ A boolean, True if the network is reserved per
iana-ipv4-special-registry or iana-ipv6-special-registry.
"""
- return (self.network_address.is_private and
- self.broadcast_address.is_private)
+ return any(self.network_address in priv_network and
+ self.broadcast_address in priv_network
+ for priv_network in self._constants._private_networks)
@property
def is_global(self):
@@ -1122,6 +1123,15 @@
return (self.network_address.is_loopback and
self.broadcast_address.is_loopback)
+
+class _BaseConstants:
+
+ _private_networks = []
+
+
+_BaseNetwork._constants = _BaseConstants
+
+
class _BaseV4:
"""Base IPv4 object.
@@ -1294,7 +1304,7 @@
# Constructing from a packed address
if isinstance(address, bytes):
self._check_packed_address(address, 4)
- self._ip = int.from_bytes(address, 'big')
+ self._ip = int.from_bytes(address) # big endian
return
# Assume input argument to be string or any object representation
@@ -1561,6 +1571,7 @@
IPv4Address._constants = _IPv4Constants
+IPv4Network._constants = _IPv4Constants
class _BaseV6:
@@ -2285,3 +2296,4 @@
IPv6Address._constants = _IPv6Constants
+IPv6Network._constants = _IPv6Constants
diff --git a/common/py3-stdlib/json/encoder.py b/common/py3-stdlib/json/encoder.py
index 687c506..45f5477 100644
--- a/common/py3-stdlib/json/encoder.py
+++ b/common/py3-stdlib/json/encoder.py
@@ -30,6 +30,7 @@
for i in range(0x20):
ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
#ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+del i
INFINITY = float('inf')
diff --git a/common/py3-stdlib/locale.py b/common/py3-stdlib/locale.py
index 6d4f519..7a7694e 100644
--- a/common/py3-stdlib/locale.py
+++ b/common/py3-stdlib/locale.py
@@ -28,7 +28,7 @@
"setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm",
"str", "atof", "atoi", "format", "format_string", "currency",
"normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY",
- "LC_NUMERIC", "LC_ALL", "CHAR_MAX"]
+ "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"]
def _strcoll(a,b):
""" strcoll(string,string) -> int.
@@ -555,6 +555,12 @@
"""
+ import warnings
+ warnings.warn(
+ "Use setlocale(), getencoding() and getlocale() instead",
+ DeprecationWarning, stacklevel=2
+ )
+
try:
# check if it's supported by the _locale module
import _locale
@@ -627,23 +633,31 @@
getdefaultlocale(). category defaults to LC_ALL.
"""
- _setlocale(category, _build_localename(getdefaultlocale()))
+ import warnings
+ warnings.warn(
+ 'Use locale.setlocale(locale.LC_ALL, "") instead',
+ DeprecationWarning, stacklevel=2
+ )
+
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ loc = getdefaultlocale()
+
+ _setlocale(category, _build_localename(loc))
try:
- from _locale import _get_locale_encoding
+ from _locale import getencoding
except ImportError:
- def _get_locale_encoding():
+ def getencoding():
if hasattr(sys, 'getandroidapilevel'):
# On Android langinfo.h and CODESET are missing, and UTF-8 is
# always used in mbstowcs() and wcstombs().
- return 'UTF-8'
- if sys.flags.utf8_mode:
- return 'UTF-8'
+ return 'utf-8'
encoding = getdefaultlocale()[1]
if encoding is None:
- # LANG not set, default conservatively to ASCII
- encoding = 'ascii'
+ # LANG not set, default to UTF-8
+ encoding = 'utf-8'
return encoding
try:
@@ -651,17 +665,30 @@
except NameError:
def getpreferredencoding(do_setlocale=True):
"""Return the charset that the user is likely using."""
- return _get_locale_encoding()
+ if sys.flags.warn_default_encoding:
+ import warnings
+ warnings.warn(
+ "UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.",
+ EncodingWarning, 2)
+ if sys.flags.utf8_mode:
+ return 'utf-8'
+ return getencoding()
else:
# On Unix, if CODESET is available, use that.
def getpreferredencoding(do_setlocale=True):
"""Return the charset that the user is likely using,
according to the system configuration."""
+
+ if sys.flags.warn_default_encoding:
+ import warnings
+ warnings.warn(
+ "UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.",
+ EncodingWarning, 2)
if sys.flags.utf8_mode:
- return 'UTF-8'
+ return 'utf-8'
if not do_setlocale:
- return _get_locale_encoding()
+ return getencoding()
old_loc = setlocale(LC_CTYPE)
try:
@@ -669,7 +696,7 @@
setlocale(LC_CTYPE, "")
except Error:
pass
- return _get_locale_encoding()
+ return getencoding()
finally:
setlocale(LC_CTYPE, old_loc)
@@ -746,6 +773,7 @@
for k, v in sorted(locale_encoding_alias.items()):
k = k.replace('_', '')
locale_encoding_alias.setdefault(k, v)
+del k, v
#
# The locale_alias table maps lowercase alias names to C locale names
diff --git a/common/py3-stdlib/logging/__init__.py b/common/py3-stdlib/logging/__init__.py
index d1d4333..bcee2ba 100644
--- a/common/py3-stdlib/logging/__init__.py
+++ b/common/py3-stdlib/logging/__init__.py
@@ -25,6 +25,7 @@
import sys, os, time, io, re, traceback, warnings, weakref, collections.abc
+from types import GenericAlias
from string import Template
from string import Formatter as StrFormatter
@@ -37,7 +38,7 @@
'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass',
'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown',
'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory',
- 'lastResort', 'raiseExceptions']
+ 'lastResort', 'raiseExceptions', 'getLevelNamesMapping']
import threading
@@ -116,6 +117,9 @@
'NOTSET': NOTSET,
}
+def getLevelNamesMapping():
+ return _nameToLevel.copy()
+
def getLevelName(level):
"""
Return the textual or numeric representation of logging level 'level'.
@@ -156,8 +160,8 @@
finally:
_releaseLock()
-if hasattr(sys, '_getframe'):
- currentframe = lambda: sys._getframe(3)
+if hasattr(sys, "_getframe"):
+ currentframe = lambda: sys._getframe(1)
else: #pragma: no cover
def currentframe():
"""Return the frame object for the caller's stack frame."""
@@ -181,13 +185,18 @@
_srcfile = os.path.normcase(addLevelName.__code__.co_filename)
# _srcfile is only used in conjunction with sys._getframe().
-# To provide compatibility with older versions of Python, set _srcfile
-# to None if _getframe() is not available; this value will prevent
-# findCaller() from being called. You can also do this if you want to avoid
-# the overhead of fetching caller information, even when _getframe() is
-# available.
-#if not hasattr(sys, '_getframe'):
-# _srcfile = None
+# Setting _srcfile to None will prevent findCaller() from being called. This
+# way, you can avoid the overhead of fetching caller information.
+
+# The following is based on warnings._is_internal_frame. It makes sure that
+# frames of the import mechanism are skipped when logging at module level and
+# using a stacklevel value greater than one.
+def _is_internal_frame(frame):
+ """Signal whether the frame is a CPython or logging module internal."""
+ filename = os.path.normcase(frame.f_code.co_filename)
+ return filename == _srcfile or (
+ "importlib" in filename and "_bootstrap" in filename
+ )
def _checkLevel(level):
@@ -845,8 +854,9 @@
if acquire and release and handlers:
acquire()
try:
- if wr in handlers:
- handlers.remove(wr)
+ handlers.remove(wr)
+ except ValueError:
+ pass
finally:
release()
@@ -1136,6 +1146,8 @@
name += ' '
return '<%s %s(%s)>' % (self.__class__.__name__, name, level)
+ __class_getitem__ = classmethod(GenericAlias)
+
class FileHandler(StreamHandler):
"""
@@ -1554,33 +1566,31 @@
f = currentframe()
#On some versions of IronPython, currentframe() returns None if
#IronPython isn't run with -X:Frames.
- if f is not None:
- f = f.f_back
- orig_f = f
- while f and stacklevel > 1:
- f = f.f_back
- stacklevel -= 1
- if not f:
- f = orig_f
- rv = "(unknown file)", 0, "(unknown function)", None
- while hasattr(f, "f_code"):
- co = f.f_code
- filename = os.path.normcase(co.co_filename)
- if filename == _srcfile:
- f = f.f_back
- continue
- sinfo = None
- if stack_info:
- sio = io.StringIO()
- sio.write('Stack (most recent call last):\n')
+ if f is None:
+ return "(unknown file)", 0, "(unknown function)", None
+ while stacklevel > 0:
+ next_f = f.f_back
+ if next_f is None:
+ ## We've got options here.
+ ## If we want to use the last (deepest) frame:
+ break
+ ## If we want to mimic the warnings module:
+ #return ("sys", 1, "(unknown function)", None)
+ ## If we want to be pedantic:
+ #raise ValueError("call stack is not deep enough")
+ f = next_f
+ if not _is_internal_frame(f):
+ stacklevel -= 1
+ co = f.f_code
+ sinfo = None
+ if stack_info:
+ with io.StringIO() as sio:
+ sio.write("Stack (most recent call last):\n")
traceback.print_stack(f, file=sio)
sinfo = sio.getvalue()
if sinfo[-1] == '\n':
sinfo = sinfo[:-1]
- sio.close()
- rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
- break
- return rv
+ return co.co_filename, f.f_lineno, co.co_name, sinfo
def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
func=None, extra=None, sinfo=None):
@@ -1767,8 +1777,6 @@
return '<%s %s (%s)>' % (self.__class__.__name__, self.name, level)
def __reduce__(self):
- # In general, only the root logger will not be accessible via its name.
- # However, the root logger's class has its own __reduce__ method.
if getLogger(self.name) is not self:
import pickle
raise pickle.PicklingError('logger cannot be pickled')
@@ -1932,6 +1940,8 @@
level = getLevelName(logger.getEffectiveLevel())
return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)
+ __class_getitem__ = classmethod(GenericAlias)
+
root = RootLogger(WARNING)
Logger.root = root
Logger.manager = Manager(Logger.root)
@@ -2242,7 +2252,9 @@
logger = getLogger("py.warnings")
if not logger.handlers:
logger.addHandler(NullHandler())
- logger.warning("%s", s)
+ # bpo-46557: Log str(s) as msg instead of logger.warning("%s", s)
+ # since some log aggregation tools group logs by the msg arg
+ logger.warning(str(s))
def captureWarnings(capture):
"""
diff --git a/common/py3-stdlib/logging/config.py b/common/py3-stdlib/logging/config.py
index 5cab008..7e78a64 100644
--- a/common/py3-stdlib/logging/config.py
+++ b/common/py3-stdlib/logging/config.py
@@ -28,9 +28,10 @@
import io
import logging
import logging.handlers
+import os
+import queue
import re
import struct
-import sys
import threading
import traceback
@@ -59,15 +60,24 @@
"""
import configparser
+ if isinstance(fname, str):
+ if not os.path.exists(fname):
+ raise FileNotFoundError(f"{fname} doesn't exist")
+ elif not os.path.getsize(fname):
+ raise RuntimeError(f'{fname} is an empty file')
+
if isinstance(fname, configparser.RawConfigParser):
cp = fname
else:
- cp = configparser.ConfigParser(defaults)
- if hasattr(fname, 'readline'):
- cp.read_file(fname)
- else:
- encoding = io.text_encoding(encoding)
- cp.read(fname, encoding=encoding)
+ try:
+ cp = configparser.ConfigParser(defaults)
+ if hasattr(fname, 'readline'):
+ cp.read_file(fname)
+ else:
+ encoding = io.text_encoding(encoding)
+ cp.read(fname, encoding=encoding)
+ except configparser.ParsingError as e:
+ raise RuntimeError(f'{fname} is invalid: {e}')
formatters = _create_formatters(cp)
@@ -392,11 +402,9 @@
self.importer(used)
found = getattr(found, frag)
return found
- except ImportError:
- e, tb = sys.exc_info()[1:]
+ except ImportError as e:
v = ValueError('Cannot resolve %r: %s' % (s, e))
- v.__cause__, v.__traceback__ = e, tb
- raise v
+ raise v from e
def ext_convert(self, value):
"""Default converter for the ext:// protocol."""
@@ -697,7 +705,11 @@
"""Add filters to a filterer from a list of names."""
for f in filters:
try:
- filterer.addFilter(self.config['filters'][f])
+ if callable(f) or callable(getattr(f, 'filter', None)):
+ filter_ = f
+ else:
+ filter_ = self.config['filters'][f]
+ filterer.addFilter(filter_)
except Exception as e:
raise ValueError('Unable to add filter %r' % f) from e
diff --git a/common/py3-stdlib/logging/handlers.py b/common/py3-stdlib/logging/handlers.py
index f0fdeda..f5a9760 100644
--- a/common/py3-stdlib/logging/handlers.py
+++ b/common/py3-stdlib/logging/handlers.py
@@ -863,12 +863,49 @@
self.address = address
self.facility = facility
self.socktype = socktype
+ self.socket = None
+ self.createSocket()
+
+ def _connect_unixsocket(self, address):
+ use_socktype = self.socktype
+ if use_socktype is None:
+ use_socktype = socket.SOCK_DGRAM
+ self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+ try:
+ self.socket.connect(address)
+ # it worked, so set self.socktype to the used type
+ self.socktype = use_socktype
+ except OSError:
+ self.socket.close()
+ if self.socktype is not None:
+ # user didn't specify falling back, so fail
+ raise
+ use_socktype = socket.SOCK_STREAM
+ self.socket = socket.socket(socket.AF_UNIX, use_socktype)
+ try:
+ self.socket.connect(address)
+ # it worked, so set self.socktype to the used type
+ self.socktype = use_socktype
+ except OSError:
+ self.socket.close()
+ raise
+
+ def createSocket(self):
+ """
+ Try to create a socket and, if it's not a datagram socket, connect it
+ to the other end. This method is called during handler initialization,
+ but it's not regarded as an error if the other end isn't listening yet
+ --- the method will be called again when emitting an event,
+ if there is no socket at that point.
+ """
+ address = self.address
+ socktype = self.socktype
if isinstance(address, str):
self.unixsocket = True
# Syslog server may be unavailable during handler initialisation.
# C's openlog() function also ignores connection errors.
- # Moreover, we ignore these errors while logging, so it not worse
+ # Moreover, we ignore these errors while logging, so it's not worse
# to ignore it also here.
try:
self._connect_unixsocket(address)
@@ -899,30 +936,6 @@
self.socket = sock
self.socktype = socktype
- def _connect_unixsocket(self, address):
- use_socktype = self.socktype
- if use_socktype is None:
- use_socktype = socket.SOCK_DGRAM
- self.socket = socket.socket(socket.AF_UNIX, use_socktype)
- try:
- self.socket.connect(address)
- # it worked, so set self.socktype to the used type
- self.socktype = use_socktype
- except OSError:
- self.socket.close()
- if self.socktype is not None:
- # user didn't specify falling back, so fail
- raise
- use_socktype = socket.SOCK_STREAM
- self.socket = socket.socket(socket.AF_UNIX, use_socktype)
- try:
- self.socket.connect(address)
- # it worked, so set self.socktype to the used type
- self.socktype = use_socktype
- except OSError:
- self.socket.close()
- raise
-
def encodePriority(self, facility, priority):
"""
Encode the facility and priority. You can pass in strings or
@@ -942,7 +955,10 @@
"""
self.acquire()
try:
- self.socket.close()
+ sock = self.socket
+ if sock:
+ self.socket = None
+ sock.close()
logging.Handler.close(self)
finally:
self.release()
@@ -982,6 +998,10 @@
# Message is a string. Convert to bytes as required by RFC 5424
msg = msg.encode('utf-8')
msg = prio + msg
+
+ if not self.socket:
+ self.createSocket()
+
if self.unixsocket:
try:
self.socket.send(msg)
@@ -1437,12 +1457,15 @@
def prepare(self, record):
"""
- Prepares a record for queuing. The object returned by this method is
+ Prepare a record for queuing. The object returned by this method is
enqueued.
- The base implementation formats the record to merge the message
- and arguments, and removes unpickleable items from the record
- in-place.
+ The base implementation formats the record to merge the message and
+ arguments, and removes unpickleable items from the record in-place.
+ Specifically, it overwrites the record's `msg` and
+ `message` attributes with the merged message (obtained by
+ calling the handler's `format` method), and sets the `args`,
+ `exc_info` and `exc_text` attributes to None.
You might want to override this method if you want to convert
the record to a dict or JSON string, or send a modified copy
diff --git a/common/py3-stdlib/mailcap.py b/common/py3-stdlib/mailcap.py
index 444c640..7278ea7 100644
--- a/common/py3-stdlib/mailcap.py
+++ b/common/py3-stdlib/mailcap.py
@@ -7,6 +7,12 @@
__all__ = ["getcaps","findmatch"]
+_DEPRECATION_MSG = ('The {name} module is deprecated and will be removed in '
+ 'Python {remove}. See the mimetypes module for an '
+ 'alternative.')
+warnings._deprecated(__name__, _DEPRECATION_MSG, remove=(3, 13))
+
+
def lineno_sort_key(entry):
# Sort in ascending order, with unspecified entries at the end
if 'lineno' in entry:
diff --git a/common/py3-stdlib/mimetypes.py b/common/py3-stdlib/mimetypes.py
index b72ce08..f6c43b3 100644
--- a/common/py3-stdlib/mimetypes.py
+++ b/common/py3-stdlib/mimetypes.py
@@ -434,6 +434,8 @@
'.doc' : 'application/msword',
'.dot' : 'application/msword',
'.wiz' : 'application/msword',
+ '.nq' : 'application/n-quads',
+ '.nt' : 'application/n-triples',
'.bin' : 'application/octet-stream',
'.a' : 'application/octet-stream',
'.dll' : 'application/octet-stream',
@@ -447,6 +449,7 @@
'.ps' : 'application/postscript',
'.ai' : 'application/postscript',
'.eps' : 'application/postscript',
+ '.trig' : 'application/trig',
'.m3u' : 'application/vnd.apple.mpegurl',
'.m3u8' : 'application/vnd.apple.mpegurl',
'.xls' : 'application/vnd.ms-excel',
@@ -514,6 +517,7 @@
'.aiff' : 'audio/x-aiff',
'.ra' : 'audio/x-pn-realaudio',
'.wav' : 'audio/x-wav',
+ '.avif' : 'image/avif',
'.bmp' : 'image/bmp',
'.gif' : 'image/gif',
'.ief' : 'image/ief',
@@ -528,7 +532,6 @@
'.tif' : 'image/tiff',
'.ico' : 'image/vnd.microsoft.icon',
'.ras' : 'image/x-cmu-raster',
- '.bmp' : 'image/x-ms-bmp',
'.pnm' : 'image/x-portable-anymap',
'.pbm' : 'image/x-portable-bitmap',
'.pgm' : 'image/x-portable-graymap',
@@ -545,14 +548,17 @@
'.csv' : 'text/csv',
'.html' : 'text/html',
'.htm' : 'text/html',
+ '.n3' : 'text/n3',
'.txt' : 'text/plain',
'.bat' : 'text/plain',
'.c' : 'text/plain',
'.h' : 'text/plain',
'.ksh' : 'text/plain',
'.pl' : 'text/plain',
+ '.srt' : 'text/plain',
'.rtx' : 'text/richtext',
'.tsv' : 'text/tab-separated-values',
+ '.vtt' : 'text/vtt',
'.py' : 'text/x-python',
'.etx' : 'text/x-setext',
'.sgm' : 'text/x-sgml',
@@ -584,6 +590,7 @@
'.pict': 'image/pict',
'.pct' : 'image/pict',
'.pic' : 'image/pict',
+ '.webp': 'image/webp',
'.xul' : 'text/xul',
}
diff --git a/common/py3-stdlib/modulefinder.py b/common/py3-stdlib/modulefinder.py
index cb455f4..a0a020f 100644
--- a/common/py3-stdlib/modulefinder.py
+++ b/common/py3-stdlib/modulefinder.py
@@ -8,14 +8,6 @@
import io
import sys
-
-LOAD_CONST = dis.opmap['LOAD_CONST']
-IMPORT_NAME = dis.opmap['IMPORT_NAME']
-STORE_NAME = dis.opmap['STORE_NAME']
-STORE_GLOBAL = dis.opmap['STORE_GLOBAL']
-STORE_OPS = STORE_NAME, STORE_GLOBAL
-EXTENDED_ARG = dis.EXTENDED_ARG
-
# Old imp constants:
_SEARCH_ERROR = 0
@@ -394,24 +386,13 @@
def scan_opcodes(self, co):
# Scan the code, and yield 'interesting' opcode combinations
- code = co.co_code
- names = co.co_names
- consts = co.co_consts
- opargs = [(op, arg) for _, op, arg in dis._unpack_opargs(code)
- if op != EXTENDED_ARG]
- for i, (op, oparg) in enumerate(opargs):
- if op in STORE_OPS:
- yield "store", (names[oparg],)
- continue
- if (op == IMPORT_NAME and i >= 2
- and opargs[i-1][0] == opargs[i-2][0] == LOAD_CONST):
- level = consts[opargs[i-2][1]]
- fromlist = consts[opargs[i-1][1]]
- if level == 0: # absolute import
- yield "absolute_import", (fromlist, names[oparg])
- else: # relative import
- yield "relative_import", (level, fromlist, names[oparg])
- continue
+ for name in dis._find_store_names(co):
+ yield "store", (name,)
+ for name, level, fromlist in dis._find_imports(co):
+ if level == 0: # absolute import
+ yield "absolute_import", (fromlist, name)
+ else: # relative import
+ yield "relative_import", (level, fromlist, name)
def scan_code(self, co, m):
code = co.co_code
diff --git a/common/py3-stdlib/multiprocessing/connection.py b/common/py3-stdlib/multiprocessing/connection.py
index 8e2facf..b08144f 100644
--- a/common/py3-stdlib/multiprocessing/connection.py
+++ b/common/py3-stdlib/multiprocessing/connection.py
@@ -183,10 +183,9 @@
self._check_closed()
self._check_writable()
m = memoryview(buf)
- # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
if m.itemsize > 1:
- m = memoryview(bytes(m))
- n = len(m)
+ m = m.cast('B')
+ n = m.nbytes
if offset < 0:
raise ValueError("offset is negative")
if n < offset:
@@ -938,7 +937,7 @@
return ready
#
-# Make connection and socket objects sharable if possible
+# Make connection and socket objects shareable if possible
#
if sys.platform == 'win32':
diff --git a/common/py3-stdlib/multiprocessing/managers.py b/common/py3-stdlib/multiprocessing/managers.py
index 22292c7..3f6479b 100644
--- a/common/py3-stdlib/multiprocessing/managers.py
+++ b/common/py3-stdlib/multiprocessing/managers.py
@@ -49,11 +49,11 @@
reduction.register(array.array, reduce_array)
view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
-if view_types[0] is not list: # only needed in Py3.0
- def rebuild_as_list(obj):
- return list, (list(obj),)
- for view_type in view_types:
- reduction.register(view_type, rebuild_as_list)
+def rebuild_as_list(obj):
+ return list, (list(obj),)
+for view_type in view_types:
+ reduction.register(view_type, rebuild_as_list)
+del view_type, view_types
#
# Type for identifying shared objects
@@ -497,7 +497,7 @@
_Server = Server
def __init__(self, address=None, authkey=None, serializer='pickle',
- ctx=None):
+ ctx=None, *, shutdown_timeout=1.0):
if authkey is None:
authkey = process.current_process().authkey
self._address = address # XXX not final address if eg ('', 0)
@@ -507,6 +507,7 @@
self._serializer = serializer
self._Listener, self._Client = listener_client[serializer]
self._ctx = ctx or get_context()
+ self._shutdown_timeout = shutdown_timeout
def get_server(self):
'''
@@ -570,8 +571,8 @@
self._state.value = State.STARTED
self.shutdown = util.Finalize(
self, type(self)._finalize_manager,
- args=(self._process, self._address, self._authkey,
- self._state, self._Client),
+ args=(self._process, self._address, self._authkey, self._state,
+ self._Client, self._shutdown_timeout),
exitpriority=0
)
@@ -656,7 +657,8 @@
self.shutdown()
@staticmethod
- def _finalize_manager(process, address, authkey, state, _Client):
+ def _finalize_manager(process, address, authkey, state, _Client,
+ shutdown_timeout):
'''
Shutdown the manager process; will be registered as a finalizer
'''
@@ -671,15 +673,17 @@
except Exception:
pass
- process.join(timeout=1.0)
+ process.join(timeout=shutdown_timeout)
if process.is_alive():
util.info('manager still alive')
if hasattr(process, 'terminate'):
util.info('trying to `terminate()` manager process')
process.terminate()
- process.join(timeout=1.0)
+ process.join(timeout=shutdown_timeout)
if process.is_alive():
util.info('manager still alive after terminate')
+ process.kill()
+ process.join()
state.value = State.SHUTDOWN
try:
@@ -1338,7 +1342,6 @@
def __del__(self):
util.debug(f"{self.__class__.__name__}.__del__ by pid {getpid()}")
- pass
def get_server(self):
'Better than monkeypatching for now; merge into Server ultimately'
diff --git a/common/py3-stdlib/multiprocessing/popen_spawn_win32.py b/common/py3-stdlib/multiprocessing/popen_spawn_win32.py
index 9c4098d..4d60ffc 100644
--- a/common/py3-stdlib/multiprocessing/popen_spawn_win32.py
+++ b/common/py3-stdlib/multiprocessing/popen_spawn_win32.py
@@ -54,19 +54,20 @@
wfd = msvcrt.open_osfhandle(whandle, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
- cmd = ' '.join('"%s"' % x for x in cmd)
python_exe = spawn.get_executable()
# bpo-35797: When running in a venv, we bypass the redirect
# executor and launch our base Python.
if WINENV and _path_eq(python_exe, sys.executable):
- python_exe = sys._base_executable
+ cmd[0] = python_exe = sys._base_executable
env = os.environ.copy()
env["__PYVENV_LAUNCHER__"] = sys.executable
else:
env = None
+ cmd = ' '.join('"%s"' % x for x in cmd)
+
with open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
diff --git a/common/py3-stdlib/multiprocessing/process.py b/common/py3-stdlib/multiprocessing/process.py
index 2d5372e..271ba3f 100644
--- a/common/py3-stdlib/multiprocessing/process.py
+++ b/common/py3-stdlib/multiprocessing/process.py
@@ -61,7 +61,7 @@
def _cleanup():
# check for processes which have finished
for p in list(_children):
- if p._popen.poll() is not None:
+ if (child_popen := p._popen) and child_popen.poll() is not None:
_children.discard(p)
#
@@ -433,6 +433,7 @@
for name, signum in list(signal.__dict__.items()):
if name[:3]=='SIG' and '_' not in name:
_exitcode_to_name[-signum] = f'-{name}'
+del name, signum
# For debug and leak testing
_dangling = WeakSet()
diff --git a/common/py3-stdlib/multiprocessing/spawn.py b/common/py3-stdlib/multiprocessing/spawn.py
index 7cc129e..09f8a22 100644
--- a/common/py3-stdlib/multiprocessing/spawn.py
+++ b/common/py3-stdlib/multiprocessing/spawn.py
@@ -33,18 +33,21 @@
WINEXE = getattr(sys, 'frozen', False)
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
-if WINSERVICE:
- _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
-else:
- _python_exe = sys.executable
-
def set_executable(exe):
global _python_exe
- _python_exe = exe
+ if sys.platform == 'win32':
+ _python_exe = os.fsdecode(exe)
+ else:
+ _python_exe = os.fsencode(exe)
def get_executable():
return _python_exe
+if WINSERVICE:
+ set_executable(os.path.join(sys.exec_prefix, 'python.exe'))
+else:
+ set_executable(sys.executable)
+
#
#
#
@@ -86,7 +89,8 @@
prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
prog %= ', '.join('%s=%r' % item for item in kwds.items())
opts = util._args_from_interpreter_flags()
- return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
+ exe = get_executable()
+ return [exe] + opts + ['-c', prog, '--multiprocessing-fork']
def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
diff --git a/common/py3-stdlib/multiprocessing/synchronize.py b/common/py3-stdlib/multiprocessing/synchronize.py
index d0be48f..42624b5 100644
--- a/common/py3-stdlib/multiprocessing/synchronize.py
+++ b/common/py3-stdlib/multiprocessing/synchronize.py
@@ -353,6 +353,9 @@
return True
return False
+ def __repr__(self) -> str:
+ set_status = 'set' if self.is_set() else 'unset'
+ return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>"
#
# Barrier
#
diff --git a/common/py3-stdlib/multiprocessing/util.py b/common/py3-stdlib/multiprocessing/util.py
index 9e07a4e..6ee0d33 100644
--- a/common/py3-stdlib/multiprocessing/util.py
+++ b/common/py3-stdlib/multiprocessing/util.py
@@ -446,13 +446,15 @@
def spawnv_passfds(path, args, passfds):
import _posixsubprocess
+ import subprocess
passfds = tuple(sorted(map(int, passfds)))
errpipe_read, errpipe_write = os.pipe()
try:
return _posixsubprocess.fork_exec(
- args, [os.fsencode(path)], True, passfds, None, None,
+ args, [path], True, passfds, None, None,
-1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
- False, False, None, None, None, -1, None)
+ False, False, -1, None, None, None, -1, None,
+ subprocess._USE_VFORK)
finally:
os.close(errpipe_read)
os.close(errpipe_write)
diff --git a/common/py3-stdlib/netrc.py b/common/py3-stdlib/netrc.py
index 734d94c..c1358aa 100644
--- a/common/py3-stdlib/netrc.py
+++ b/common/py3-stdlib/netrc.py
@@ -19,6 +19,50 @@
return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno)
+class _netrclex:
+ def __init__(self, fp):
+ self.lineno = 1
+ self.instream = fp
+ self.whitespace = "\n\t\r "
+ self.pushback = []
+
+ def _read_char(self):
+ ch = self.instream.read(1)
+ if ch == "\n":
+ self.lineno += 1
+ return ch
+
+ def get_token(self):
+ if self.pushback:
+ return self.pushback.pop(0)
+ token = ""
+ fiter = iter(self._read_char, "")
+ for ch in fiter:
+ if ch in self.whitespace:
+ continue
+ if ch == '"':
+ for ch in fiter:
+ if ch == '"':
+ return token
+ elif ch == "\\":
+ ch = self._read_char()
+ token += ch
+ else:
+ if ch == "\\":
+ ch = self._read_char()
+ token += ch
+ for ch in fiter:
+ if ch in self.whitespace:
+ return token
+ elif ch == "\\":
+ ch = self._read_char()
+ token += ch
+ return token
+
+ def push_token(self, token):
+ self.pushback.append(token)
+
+
class netrc:
def __init__(self, file=None):
default_netrc = file is None
@@ -34,9 +78,7 @@
self._parse(file, fp, default_netrc)
def _parse(self, file, fp, default_netrc):
- lexer = shlex.shlex(fp)
- lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
- lexer.commenters = lexer.commenters.replace('#', '')
+ lexer = _netrclex(fp)
while 1:
# Look for a machine, default, or macdef top-level keyword
saved_lineno = lexer.lineno
@@ -51,14 +93,19 @@
entryname = lexer.get_token()
elif tt == 'default':
entryname = 'default'
- elif tt == 'macdef': # Just skip to end of macdefs
+ elif tt == 'macdef':
entryname = lexer.get_token()
self.macros[entryname] = []
- lexer.whitespace = ' \t'
while 1:
line = lexer.instream.readline()
- if not line or line == '\012':
- lexer.whitespace = ' \t\r\n'
+ if not line:
+ raise NetrcParseError(
+ "Macro definition missing null line terminator.",
+ file, lexer.lineno)
+ if line == '\n':
+ # a macro definition finished with consecutive new-line
+ # characters. The first \n is encountered by the
+ # readline() method and this is the second \n.
break
self.macros[entryname].append(line)
continue
@@ -66,53 +113,55 @@
raise NetrcParseError(
"bad toplevel token %r" % tt, file, lexer.lineno)
+ if not entryname:
+ raise NetrcParseError("missing %r name" % tt, file, lexer.lineno)
+
# We're looking at start of an entry for a named machine or default.
- login = ''
- account = password = None
+ login = account = password = ''
self.hosts[entryname] = {}
while 1:
+ prev_lineno = lexer.lineno
tt = lexer.get_token()
- if (tt.startswith('#') or
- tt in {'', 'machine', 'default', 'macdef'}):
- if password:
- self.hosts[entryname] = (login, account, password)
- lexer.push_token(tt)
- break
- else:
- raise NetrcParseError(
- "malformed %s entry %s terminated by %s"
- % (toplevel, entryname, repr(tt)),
- file, lexer.lineno)
+ if tt.startswith('#'):
+ if lexer.lineno == prev_lineno:
+ lexer.instream.readline()
+ continue
+ if tt in {'', 'machine', 'default', 'macdef'}:
+ self.hosts[entryname] = (login, account, password)
+ lexer.push_token(tt)
+ break
elif tt == 'login' or tt == 'user':
login = lexer.get_token()
elif tt == 'account':
account = lexer.get_token()
elif tt == 'password':
- if os.name == 'posix' and default_netrc:
- prop = os.fstat(fp.fileno())
- if prop.st_uid != os.getuid():
- import pwd
- try:
- fowner = pwd.getpwuid(prop.st_uid)[0]
- except KeyError:
- fowner = 'uid %s' % prop.st_uid
- try:
- user = pwd.getpwuid(os.getuid())[0]
- except KeyError:
- user = 'uid %s' % os.getuid()
- raise NetrcParseError(
- ("~/.netrc file owner (%s) does not match"
- " current user (%s)") % (fowner, user),
- file, lexer.lineno)
- if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
- raise NetrcParseError(
- "~/.netrc access too permissive: access"
- " permissions must restrict access to only"
- " the owner", file, lexer.lineno)
password = lexer.get_token()
else:
raise NetrcParseError("bad follower token %r" % tt,
file, lexer.lineno)
+ self._security_check(fp, default_netrc, self.hosts[entryname][0])
+
+ def _security_check(self, fp, default_netrc, login):
+ if os.name == 'posix' and default_netrc and login != "anonymous":
+ prop = os.fstat(fp.fileno())
+ if prop.st_uid != os.getuid():
+ import pwd
+ try:
+ fowner = pwd.getpwuid(prop.st_uid)[0]
+ except KeyError:
+ fowner = 'uid %s' % prop.st_uid
+ try:
+ user = pwd.getpwuid(os.getuid())[0]
+ except KeyError:
+ user = 'uid %s' % os.getuid()
+ raise NetrcParseError(
+ (f"~/.netrc file owner ({fowner}, {user}) does not match"
+ " current user"))
+ if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
+ raise NetrcParseError(
+ "~/.netrc access too permissive: access"
+ " permissions must restrict access to only"
+ " the owner")
def authenticators(self, host):
"""Return a (user, account, password) tuple for given host."""
diff --git a/common/py3-stdlib/nntplib.py b/common/py3-stdlib/nntplib.py
index f6e746e..dddea05 100644
--- a/common/py3-stdlib/nntplib.py
+++ b/common/py3-stdlib/nntplib.py
@@ -68,6 +68,7 @@
import collections
import datetime
import sys
+import warnings
try:
import ssl
@@ -85,6 +86,8 @@
"decode_header",
]
+warnings._deprecated(__name__, remove=(3, 13))
+
# maximal line length when calling readline(). This is to prevent
# reading arbitrary length lines. RFC 3977 limits NNTP line length to
# 512 characters, including CRLF. We have selected 2048 just to be on
diff --git a/common/py3-stdlib/ntpath.py b/common/py3-stdlib/ntpath.py
index c14e5c7..0444b0f 100644
--- a/common/py3-stdlib/ntpath.py
+++ b/common/py3-stdlib/ntpath.py
@@ -87,16 +87,20 @@
def isabs(s):
"""Test whether a path is absolute"""
s = os.fspath(s)
- # Paths beginning with \\?\ are always absolute, but do not
- # necessarily contain a drive.
if isinstance(s, bytes):
- if s.replace(b'/', b'\\').startswith(b'\\\\?\\'):
- return True
+ sep = b'\\'
+ altsep = b'/'
+ colon_sep = b':\\'
else:
- if s.replace('/', '\\').startswith('\\\\?\\'):
- return True
- s = splitdrive(s)[1]
- return len(s) > 0 and s[0] in _get_bothseps(s)
+ sep = '\\'
+ altsep = '/'
+ colon_sep = ':\\'
+ s = s[:3].replace(altsep, sep)
+ # Absolute: UNC, device, and paths with a drive and root.
+ # LEGACY BUG: isabs("/x") should be false since the path has no drive.
+ if s.startswith(sep) or s.startswith(colon_sep, 1):
+ return True
+ return False
# Join two (or more) paths.
@@ -172,28 +176,26 @@
sep = b'\\'
altsep = b'/'
colon = b':'
+ unc_prefix = b'\\\\?\\UNC\\'
else:
sep = '\\'
altsep = '/'
colon = ':'
+ unc_prefix = '\\\\?\\UNC\\'
normp = p.replace(altsep, sep)
- if (normp[0:2] == sep*2) and (normp[2:3] != sep):
- # is a UNC path:
- # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path
- # \\machine\mountpoint\directory\etc\...
- # directory ^^^^^^^^^^^^^^^
- index = normp.find(sep, 2)
+ if normp[0:2] == sep * 2:
+ # UNC drives, e.g. \\server\share or \\?\UNC\server\share
+ # Device drives, e.g. \\.\device or \\?\device
+ start = 8 if normp[:8].upper() == unc_prefix else 2
+ index = normp.find(sep, start)
if index == -1:
- return p[:0], p
+ return p, p[:0]
index2 = normp.find(sep, index + 1)
- # a UNC path can't have two slashes in a row
- # (after the initial two)
- if index2 == index + 1:
- return p[:0], p
if index2 == -1:
- index2 = len(p)
+ return p, p[:0]
return p[:index2], p[index2:]
if normp[1:2] == colon:
+ # Drive-letter drives, e.g. X:
return p[:2], p[2:]
return p[:0], p
@@ -294,11 +296,13 @@
root, rest = splitdrive(path)
if root and root[0] in seps:
return (not rest) or (rest in seps)
- if rest in seps:
+ if rest and rest in seps:
return True
if _getvolumepathname:
- return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps)
+ x = path.rstrip(seps)
+ y =_getvolumepathname(path).rstrip(seps)
+ return x.casefold() == y.casefold()
else:
return False
@@ -485,56 +489,59 @@
# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
# Previously, this function also truncated pathnames to 8+3 format,
# but as this module is called "ntpath", that's obviously wrong!
+try:
+ from nt import _path_normpath
-def normpath(path):
- """Normalize path, eliminating double slashes, etc."""
- path = os.fspath(path)
- if isinstance(path, bytes):
- sep = b'\\'
- altsep = b'/'
- curdir = b'.'
- pardir = b'..'
- special_prefixes = (b'\\\\.\\', b'\\\\?\\')
- else:
- sep = '\\'
- altsep = '/'
- curdir = '.'
- pardir = '..'
- special_prefixes = ('\\\\.\\', '\\\\?\\')
- if path.startswith(special_prefixes):
- # in the case of paths with these prefixes:
- # \\.\ -> device names
- # \\?\ -> literal paths
- # do not do any normalization, but return the path
- # unchanged apart from the call to os.fspath()
- return path
- path = path.replace(altsep, sep)
- prefix, path = splitdrive(path)
+except ImportError:
+ def normpath(path):
+ """Normalize path, eliminating double slashes, etc."""
+ path = os.fspath(path)
+ if isinstance(path, bytes):
+ sep = b'\\'
+ altsep = b'/'
+ curdir = b'.'
+ pardir = b'..'
+ else:
+ sep = '\\'
+ altsep = '/'
+ curdir = '.'
+ pardir = '..'
+ path = path.replace(altsep, sep)
+ prefix, path = splitdrive(path)
- # collapse initial backslashes
- if path.startswith(sep):
- prefix += sep
- path = path.lstrip(sep)
+ # collapse initial backslashes
+ if path.startswith(sep):
+ prefix += sep
+ path = path.lstrip(sep)
- comps = path.split(sep)
- i = 0
- while i < len(comps):
- if not comps[i] or comps[i] == curdir:
- del comps[i]
- elif comps[i] == pardir:
- if i > 0 and comps[i-1] != pardir:
- del comps[i-1:i+1]
- i -= 1
- elif i == 0 and prefix.endswith(sep):
+ comps = path.split(sep)
+ i = 0
+ while i < len(comps):
+ if not comps[i] or comps[i] == curdir:
del comps[i]
+ elif comps[i] == pardir:
+ if i > 0 and comps[i-1] != pardir:
+ del comps[i-1:i+1]
+ i -= 1
+ elif i == 0 and prefix.endswith(sep):
+ del comps[i]
+ else:
+ i += 1
else:
i += 1
- else:
- i += 1
- # If the path is now empty, substitute '.'
- if not prefix and not comps:
- comps.append(curdir)
- return prefix + sep.join(comps)
+ # If the path is now empty, substitute '.'
+ if not prefix and not comps:
+ comps.append(curdir)
+ return prefix + sep.join(comps)
+
+else:
+ def normpath(path):
+ """Normalize path, eliminating double slashes, etc."""
+ path = os.fspath(path)
+ if isinstance(path, bytes):
+ return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
+ return _path_normpath(path) or "."
+
def _abspath_fallback(path):
"""Return the absolute version of a path as a fallback function in case
@@ -563,7 +570,7 @@
def abspath(path):
"""Return the absolute version of a path."""
try:
- return normpath(_getfullpathname(path))
+ return _getfullpathname(normpath(path))
except (OSError, ValueError):
return _abspath_fallback(path)
@@ -637,7 +644,7 @@
# Non-strict algorithm is to find as much of the target directory
# as we can and join the rest.
- tail = ''
+ tail = path[:0]
while path:
try:
path = _getfinalpathname(path)
diff --git a/common/py3-stdlib/opcode.py b/common/py3-stdlib/opcode.py
index 37e88e9..bc3c02a 100644
--- a/common/py3-stdlib/opcode.py
+++ b/common/py3-stdlib/opcode.py
@@ -54,12 +54,9 @@
# Instruction opcodes for compiled code
# Blank lines correspond to available opcodes
+def_op('CACHE', 0)
def_op('POP_TOP', 1)
-def_op('ROT_TWO', 2)
-def_op('ROT_THREE', 3)
-def_op('DUP_TOP', 4)
-def_op('DUP_TOP_TWO', 5)
-def_op('ROT_FOUR', 6)
+def_op('PUSH_NULL', 2)
def_op('NOP', 9)
def_op('UNARY_POSITIVE', 10)
@@ -67,65 +64,43 @@
def_op('UNARY_NOT', 12)
def_op('UNARY_INVERT', 15)
-def_op('BINARY_MATRIX_MULTIPLY', 16)
-def_op('INPLACE_MATRIX_MULTIPLY', 17)
-def_op('BINARY_POWER', 19)
-def_op('BINARY_MULTIPLY', 20)
-
-def_op('BINARY_MODULO', 22)
-def_op('BINARY_ADD', 23)
-def_op('BINARY_SUBTRACT', 24)
def_op('BINARY_SUBSCR', 25)
-def_op('BINARY_FLOOR_DIVIDE', 26)
-def_op('BINARY_TRUE_DIVIDE', 27)
-def_op('INPLACE_FLOOR_DIVIDE', 28)
-def_op('INPLACE_TRUE_DIVIDE', 29)
+
def_op('GET_LEN', 30)
def_op('MATCH_MAPPING', 31)
def_op('MATCH_SEQUENCE', 32)
def_op('MATCH_KEYS', 33)
-def_op('COPY_DICT_WITHOUT_KEYS', 34)
+
+def_op('PUSH_EXC_INFO', 35)
+def_op('CHECK_EXC_MATCH', 36)
+def_op('CHECK_EG_MATCH', 37)
def_op('WITH_EXCEPT_START', 49)
def_op('GET_AITER', 50)
def_op('GET_ANEXT', 51)
def_op('BEFORE_ASYNC_WITH', 52)
-
+def_op('BEFORE_WITH', 53)
def_op('END_ASYNC_FOR', 54)
-def_op('INPLACE_ADD', 55)
-def_op('INPLACE_SUBTRACT', 56)
-def_op('INPLACE_MULTIPLY', 57)
-def_op('INPLACE_MODULO', 59)
def_op('STORE_SUBSCR', 60)
def_op('DELETE_SUBSCR', 61)
-def_op('BINARY_LSHIFT', 62)
-def_op('BINARY_RSHIFT', 63)
-def_op('BINARY_AND', 64)
-def_op('BINARY_XOR', 65)
-def_op('BINARY_OR', 66)
-def_op('INPLACE_POWER', 67)
+
def_op('GET_ITER', 68)
def_op('GET_YIELD_FROM_ITER', 69)
def_op('PRINT_EXPR', 70)
def_op('LOAD_BUILD_CLASS', 71)
-def_op('YIELD_FROM', 72)
-def_op('GET_AWAITABLE', 73)
+
def_op('LOAD_ASSERTION_ERROR', 74)
-def_op('INPLACE_LSHIFT', 75)
-def_op('INPLACE_RSHIFT', 76)
-def_op('INPLACE_AND', 77)
-def_op('INPLACE_XOR', 78)
-def_op('INPLACE_OR', 79)
+def_op('RETURN_GENERATOR', 75)
def_op('LIST_TO_TUPLE', 82)
def_op('RETURN_VALUE', 83)
def_op('IMPORT_STAR', 84)
def_op('SETUP_ANNOTATIONS', 85)
def_op('YIELD_VALUE', 86)
-def_op('POP_BLOCK', 87)
-
+def_op('ASYNC_GEN_WRAP', 87)
+def_op('PREP_RERAISE_STAR', 88)
def_op('POP_EXCEPT', 89)
HAVE_ARGUMENT = 90 # Opcodes from here have an argument:
@@ -139,7 +114,7 @@
name_op('DELETE_ATTR', 96) # ""
name_op('STORE_GLOBAL', 97) # ""
name_op('DELETE_GLOBAL', 98) # ""
-def_op('ROT_N', 99)
+def_op('SWAP', 99)
def_op('LOAD_CONST', 100) # Index in const list
hasconst.append(100)
name_op('LOAD_NAME', 101) # Index in name list
@@ -152,45 +127,45 @@
hascompare.append(107)
name_op('IMPORT_NAME', 108) # Index in name list
name_op('IMPORT_FROM', 109) # Index in name list
-jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip
-jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code
-jabs_op('JUMP_IF_TRUE_OR_POP', 112) # ""
-jabs_op('JUMP_ABSOLUTE', 113) # ""
-jabs_op('POP_JUMP_IF_FALSE', 114) # ""
-jabs_op('POP_JUMP_IF_TRUE', 115) # ""
+jrel_op('JUMP_FORWARD', 110) # Number of words to skip
+jrel_op('JUMP_IF_FALSE_OR_POP', 111) # Number of words to skip
+jrel_op('JUMP_IF_TRUE_OR_POP', 112) # ""
+jrel_op('POP_JUMP_FORWARD_IF_FALSE', 114)
+jrel_op('POP_JUMP_FORWARD_IF_TRUE', 115)
name_op('LOAD_GLOBAL', 116) # Index in name list
def_op('IS_OP', 117)
def_op('CONTAINS_OP', 118)
def_op('RERAISE', 119)
-
-jabs_op('JUMP_IF_NOT_EXC_MATCH', 121)
-jrel_op('SETUP_FINALLY', 122) # Distance to target address
-
+def_op('COPY', 120)
+def_op('BINARY_OP', 122)
+jrel_op('SEND', 123) # Number of bytes to skip
def_op('LOAD_FAST', 124) # Local variable number
haslocal.append(124)
def_op('STORE_FAST', 125) # Local variable number
haslocal.append(125)
def_op('DELETE_FAST', 126) # Local variable number
haslocal.append(126)
-
-def_op('GEN_START', 129) # Kind of generator/coroutine
+jrel_op('POP_JUMP_FORWARD_IF_NOT_NONE', 128)
+jrel_op('POP_JUMP_FORWARD_IF_NONE', 129)
def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3)
-def_op('CALL_FUNCTION', 131) # #args
+def_op('GET_AWAITABLE', 131)
def_op('MAKE_FUNCTION', 132) # Flags
def_op('BUILD_SLICE', 133) # Number of items
-
-def_op('LOAD_CLOSURE', 135)
+jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards)
+def_op('MAKE_CELL', 135)
hasfree.append(135)
-def_op('LOAD_DEREF', 136)
+def_op('LOAD_CLOSURE', 136)
hasfree.append(136)
-def_op('STORE_DEREF', 137)
+def_op('LOAD_DEREF', 137)
hasfree.append(137)
-def_op('DELETE_DEREF', 138)
+def_op('STORE_DEREF', 138)
hasfree.append(138)
+def_op('DELETE_DEREF', 139)
+hasfree.append(139)
+jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards)
-def_op('CALL_FUNCTION_KW', 141) # #args + #kwargs
def_op('CALL_FUNCTION_EX', 142) # Flags
-jrel_op('SETUP_WITH', 143)
+
def_op('EXTENDED_ARG', 144)
EXTENDED_ARG = 144
def_op('LIST_APPEND', 145)
@@ -198,19 +173,235 @@
def_op('MAP_ADD', 147)
def_op('LOAD_CLASSDEREF', 148)
hasfree.append(148)
+def_op('COPY_FREE_VARS', 149)
+def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py
def_op('MATCH_CLASS', 152)
-jrel_op('SETUP_ASYNC_WITH', 154)
def_op('FORMAT_VALUE', 155)
def_op('BUILD_CONST_KEY_MAP', 156)
def_op('BUILD_STRING', 157)
name_op('LOAD_METHOD', 160)
-def_op('CALL_METHOD', 161)
+
def_op('LIST_EXTEND', 162)
def_op('SET_UPDATE', 163)
def_op('DICT_MERGE', 164)
def_op('DICT_UPDATE', 165)
+def_op('PRECALL', 166)
+
+def_op('CALL', 171)
+def_op('KW_NAMES', 172)
+hasconst.append(172)
+
+jrel_op('POP_JUMP_BACKWARD_IF_NOT_NONE', 173)
+jrel_op('POP_JUMP_BACKWARD_IF_NONE', 174)
+jrel_op('POP_JUMP_BACKWARD_IF_FALSE', 175)
+jrel_op('POP_JUMP_BACKWARD_IF_TRUE', 176)
+
del def_op, name_op, jrel_op, jabs_op
+
+_nb_ops = [
+ ("NB_ADD", "+"),
+ ("NB_AND", "&"),
+ ("NB_FLOOR_DIVIDE", "//"),
+ ("NB_LSHIFT", "<<"),
+ ("NB_MATRIX_MULTIPLY", "@"),
+ ("NB_MULTIPLY", "*"),
+ ("NB_REMAINDER", "%"),
+ ("NB_OR", "|"),
+ ("NB_POWER", "**"),
+ ("NB_RSHIFT", ">>"),
+ ("NB_SUBTRACT", "-"),
+ ("NB_TRUE_DIVIDE", "/"),
+ ("NB_XOR", "^"),
+ ("NB_INPLACE_ADD", "+="),
+ ("NB_INPLACE_AND", "&="),
+ ("NB_INPLACE_FLOOR_DIVIDE", "//="),
+ ("NB_INPLACE_LSHIFT", "<<="),
+ ("NB_INPLACE_MATRIX_MULTIPLY", "@="),
+ ("NB_INPLACE_MULTIPLY", "*="),
+ ("NB_INPLACE_REMAINDER", "%="),
+ ("NB_INPLACE_OR", "|="),
+ ("NB_INPLACE_POWER", "**="),
+ ("NB_INPLACE_RSHIFT", ">>="),
+ ("NB_INPLACE_SUBTRACT", "-="),
+ ("NB_INPLACE_TRUE_DIVIDE", "/="),
+ ("NB_INPLACE_XOR", "^="),
+]
+
+_specializations = {
+ "BINARY_OP": [
+ "BINARY_OP_ADAPTIVE",
+ "BINARY_OP_ADD_FLOAT",
+ "BINARY_OP_ADD_INT",
+ "BINARY_OP_ADD_UNICODE",
+ "BINARY_OP_INPLACE_ADD_UNICODE",
+ "BINARY_OP_MULTIPLY_FLOAT",
+ "BINARY_OP_MULTIPLY_INT",
+ "BINARY_OP_SUBTRACT_FLOAT",
+ "BINARY_OP_SUBTRACT_INT",
+ ],
+ "BINARY_SUBSCR": [
+ "BINARY_SUBSCR_ADAPTIVE",
+ "BINARY_SUBSCR_DICT",
+ "BINARY_SUBSCR_GETITEM",
+ "BINARY_SUBSCR_LIST_INT",
+ "BINARY_SUBSCR_TUPLE_INT",
+ ],
+ "CALL": [
+ "CALL_ADAPTIVE",
+ "CALL_PY_EXACT_ARGS",
+ "CALL_PY_WITH_DEFAULTS",
+ ],
+ "COMPARE_OP": [
+ "COMPARE_OP_ADAPTIVE",
+ "COMPARE_OP_FLOAT_JUMP",
+ "COMPARE_OP_INT_JUMP",
+ "COMPARE_OP_STR_JUMP",
+ ],
+ "EXTENDED_ARG": [
+ "EXTENDED_ARG_QUICK",
+ ],
+ "JUMP_BACKWARD": [
+ "JUMP_BACKWARD_QUICK",
+ ],
+ "LOAD_ATTR": [
+ "LOAD_ATTR_ADAPTIVE",
+ "LOAD_ATTR_INSTANCE_VALUE",
+ "LOAD_ATTR_MODULE",
+ "LOAD_ATTR_SLOT",
+ "LOAD_ATTR_WITH_HINT",
+ ],
+ "LOAD_CONST": [
+ "LOAD_CONST__LOAD_FAST",
+ ],
+ "LOAD_FAST": [
+ "LOAD_FAST__LOAD_CONST",
+ "LOAD_FAST__LOAD_FAST",
+ ],
+ "LOAD_GLOBAL": [
+ "LOAD_GLOBAL_ADAPTIVE",
+ "LOAD_GLOBAL_BUILTIN",
+ "LOAD_GLOBAL_MODULE",
+ ],
+ "LOAD_METHOD": [
+ "LOAD_METHOD_ADAPTIVE",
+ "LOAD_METHOD_CLASS",
+ "LOAD_METHOD_MODULE",
+ "LOAD_METHOD_NO_DICT",
+ "LOAD_METHOD_WITH_DICT",
+ "LOAD_METHOD_WITH_VALUES",
+ ],
+ "PRECALL": [
+ "PRECALL_ADAPTIVE",
+ "PRECALL_BOUND_METHOD",
+ "PRECALL_BUILTIN_CLASS",
+ "PRECALL_BUILTIN_FAST_WITH_KEYWORDS",
+ "PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS",
+ "PRECALL_NO_KW_BUILTIN_FAST",
+ "PRECALL_NO_KW_BUILTIN_O",
+ "PRECALL_NO_KW_ISINSTANCE",
+ "PRECALL_NO_KW_LEN",
+ "PRECALL_NO_KW_LIST_APPEND",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_O",
+ "PRECALL_NO_KW_STR_1",
+ "PRECALL_NO_KW_TUPLE_1",
+ "PRECALL_NO_KW_TYPE_1",
+ "PRECALL_PYFUNC",
+ ],
+ "RESUME": [
+ "RESUME_QUICK",
+ ],
+ "STORE_ATTR": [
+ "STORE_ATTR_ADAPTIVE",
+ "STORE_ATTR_INSTANCE_VALUE",
+ "STORE_ATTR_SLOT",
+ "STORE_ATTR_WITH_HINT",
+ ],
+ "STORE_FAST": [
+ "STORE_FAST__LOAD_FAST",
+ "STORE_FAST__STORE_FAST",
+ ],
+ "STORE_SUBSCR": [
+ "STORE_SUBSCR_ADAPTIVE",
+ "STORE_SUBSCR_DICT",
+ "STORE_SUBSCR_LIST_INT",
+ ],
+ "UNPACK_SEQUENCE": [
+ "UNPACK_SEQUENCE_ADAPTIVE",
+ "UNPACK_SEQUENCE_LIST",
+ "UNPACK_SEQUENCE_TUPLE",
+ "UNPACK_SEQUENCE_TWO_TUPLE",
+ ],
+}
+_specialized_instructions = [
+ opcode for family in _specializations.values() for opcode in family
+]
+_specialization_stats = [
+ "success",
+ "failure",
+ "hit",
+ "deferred",
+ "miss",
+ "deopt",
+]
+
+_cache_format = {
+ "LOAD_GLOBAL": {
+ "counter": 1,
+ "index": 1,
+ "module_keys_version": 2,
+ "builtin_keys_version": 1,
+ },
+ "BINARY_OP": {
+ "counter": 1,
+ },
+ "UNPACK_SEQUENCE": {
+ "counter": 1,
+ },
+ "COMPARE_OP": {
+ "counter": 1,
+ "mask": 1,
+ },
+ "BINARY_SUBSCR": {
+ "counter": 1,
+ "type_version": 2,
+ "func_version": 1,
+ },
+ "LOAD_ATTR": {
+ "counter": 1,
+ "version": 2,
+ "index": 1,
+ },
+ "STORE_ATTR": {
+ "counter": 1,
+ "version": 2,
+ "index": 1,
+ },
+ "LOAD_METHOD": {
+ "counter": 1,
+ "type_version": 2,
+ "dict_offset": 1,
+ "keys_version": 2,
+ "descr": 4,
+ },
+ "CALL": {
+ "counter": 1,
+ "func_version": 2,
+ "min_args": 1,
+ },
+ "PRECALL": {
+ "counter": 1,
+ },
+ "STORE_SUBSCR": {
+ "counter": 1,
+ },
+}
+
+_inline_cache_entries = [
+ sum(_cache_format.get(opname[opcode], {}).values()) for opcode in range(256)
+]
diff --git a/common/py3-stdlib/operator.py b/common/py3-stdlib/operator.py
index 241fdbb..30116c1 100644
--- a/common/py3-stdlib/operator.py
+++ b/common/py3-stdlib/operator.py
@@ -10,7 +10,7 @@
This is the pure Python implementation of the module.
"""
-__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf',
+__all__ = ['abs', 'add', 'and_', 'attrgetter', 'call', 'concat', 'contains', 'countOf',
'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand',
'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul',
'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift',
@@ -221,6 +221,12 @@
raise ValueError(msg)
return val
+# Other Operations ************************************************************#
+
+def call(obj, /, *args, **kwargs):
+ """Same as obj(*args, **kwargs)."""
+ return obj(*args, **kwargs)
+
# Generalized Lookup Objects **************************************************#
class attrgetter:
@@ -423,6 +429,7 @@
__abs__ = abs
__add__ = add
__and__ = and_
+__call__ = call
__floordiv__ = floordiv
__index__ = index
__inv__ = inv
diff --git a/common/py3-stdlib/os.py b/common/py3-stdlib/os.py
index 4f2ffce..fd1e774 100644
--- a/common/py3-stdlib/os.py
+++ b/common/py3-stdlib/os.py
@@ -332,8 +332,8 @@
import os
from os.path import join, getsize
for root, dirs, files in os.walk('python/Lib/email'):
- print(root, "consumes", end="")
- print(sum(getsize(join(root, name)) for name in files), end="")
+ print(root, "consumes ")
+ print(sum(getsize(join(root, name)) for name in files), end=" ")
print("bytes in", len(files), "non-directory files")
if 'CVS' in dirs:
dirs.remove('CVS') # don't visit CVS directories
@@ -462,8 +462,7 @@
dirs.remove('CVS') # don't visit CVS directories
"""
sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd)
- if not isinstance(top, int) or not hasattr(top, '__index__'):
- top = fspath(top)
+ top = fspath(top)
# Note: To guard against symlink races, we use the standard
# lstat()/open()/fstat() trick.
if not follow_symlinks:
@@ -705,9 +704,11 @@
return len(self._data)
def __repr__(self):
- return 'environ({{{}}})'.format(', '.join(
- ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value))
- for key, value in self._data.items())))
+ formatted_items = ", ".join(
+ f"{self.decodekey(key)!r}: {self.decodevalue(value)!r}"
+ for key, value in self._data.items()
+ )
+ return f"environ({{{formatted_items}}})"
def copy(self):
return dict(self)
@@ -981,7 +982,7 @@
raise ValueError("invalid mode %r" % mode)
if buffering == 0 or buffering is None:
raise ValueError("popen() does not support unbuffered streams")
- import subprocess, io
+ import subprocess
if mode == "r":
proc = subprocess.Popen(cmd,
shell=True, text=True,
diff --git a/common/py3-stdlib/pathlib.py b/common/py3-stdlib/pathlib.py
index 97b23ca..ecb1e8a 100644
--- a/common/py3-stdlib/pathlib.py
+++ b/common/py3-stdlib/pathlib.py
@@ -8,7 +8,7 @@
import sys
import warnings
from _collections_abc import Sequence
-from errno import EINVAL, ENOENT, ENOTDIR, EBADF, ELOOP
+from errno import ENOENT, ENOTDIR, EBADF, ELOOP
from operator import attrgetter
from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO
from urllib.parse import quote_from_bytes as urlquote_from_bytes
@@ -28,7 +28,7 @@
_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself
# EBADF - guard against macOS `stat` throwing EBADF
-_IGNORED_ERROS = (ENOENT, ENOTDIR, EBADF, ELOOP)
+_IGNORED_ERRNOS = (ENOENT, ENOTDIR, EBADF, ELOOP)
_IGNORED_WINERRORS = (
_WINERROR_NOT_READY,
@@ -36,7 +36,7 @@
_WINERROR_CANT_RESOLVE_FILENAME)
def _ignore_error(exception):
- return (getattr(exception, 'errno', None) in _IGNORED_ERROS or
+ return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or
getattr(exception, 'winerror', None) in _IGNORED_WINERRORS)
@@ -274,93 +274,6 @@
_posix_flavour = _PosixFlavour()
-class _Accessor:
- """An accessor implements a particular (system-specific or not) way of
- accessing paths on the filesystem."""
-
-
-class _NormalAccessor(_Accessor):
-
- stat = os.stat
-
- open = io.open
-
- listdir = os.listdir
-
- scandir = os.scandir
-
- chmod = os.chmod
-
- mkdir = os.mkdir
-
- unlink = os.unlink
-
- if hasattr(os, "link"):
- link = os.link
- else:
- def link(self, src, dst):
- raise NotImplementedError("os.link() not available on this system")
-
- rmdir = os.rmdir
-
- rename = os.rename
-
- replace = os.replace
-
- if hasattr(os, "symlink"):
- symlink = os.symlink
- else:
- def symlink(self, src, dst, target_is_directory=False):
- raise NotImplementedError("os.symlink() not available on this system")
-
- def touch(self, path, mode=0o666, exist_ok=True):
- if exist_ok:
- # First try to bump modification time
- # Implementation note: GNU touch uses the UTIME_NOW option of
- # the utimensat() / futimens() functions.
- try:
- os.utime(path, None)
- except OSError:
- # Avoid exception chaining
- pass
- else:
- return
- flags = os.O_CREAT | os.O_WRONLY
- if not exist_ok:
- flags |= os.O_EXCL
- fd = os.open(path, flags, mode)
- os.close(fd)
-
- if hasattr(os, "readlink"):
- readlink = os.readlink
- else:
- def readlink(self, path):
- raise NotImplementedError("os.readlink() not available on this system")
-
- def owner(self, path):
- try:
- import pwd
- return pwd.getpwuid(self.stat(path).st_uid).pw_name
- except ImportError:
- raise NotImplementedError("Path.owner() is unsupported on this system")
-
- def group(self, path):
- try:
- import grp
- return grp.getgrgid(self.stat(path).st_gid).gr_name
- except ImportError:
- raise NotImplementedError("Path.group() is unsupported on this system")
-
- getcwd = os.getcwd
-
- expanduser = staticmethod(os.path.expanduser)
-
- realpath = staticmethod(os.path.realpath)
-
-
-_normal_accessor = _NormalAccessor()
-
-
#
# Globbing helpers
#
@@ -368,6 +281,8 @@
def _make_selector(pattern_parts, flavour):
pat = pattern_parts[0]
child_parts = pattern_parts[1:]
+ if not pat:
+ return _TerminatingSelector()
if pat == '**':
cls = _RecursiveWildcardSelector
elif '**' in pat:
@@ -401,7 +316,7 @@
path_cls = type(parent_path)
is_dir = path_cls.is_dir
exists = path_cls.exists
- scandir = parent_path._accessor.scandir
+ scandir = path_cls._scandir
if not is_dir(parent_path):
return iter([])
return self._select_from(parent_path, is_dir, exists, scandir)
@@ -473,11 +388,11 @@
for entry in entries:
entry_is_dir = False
try:
- entry_is_dir = entry.is_dir()
+ entry_is_dir = entry.is_dir(follow_symlinks=False)
except OSError as e:
if not _ignore_error(e):
raise
- if entry_is_dir and not entry.is_symlink():
+ if entry_is_dir:
path = parent_path._make_child_relpath(entry.name)
for p in self._iterate_directories(path, is_dir, scandir):
yield p
@@ -692,9 +607,6 @@
return NotImplemented
return self._cparts >= other._cparts
- def __class_getitem__(cls, type):
- return cls
-
drive = property(attrgetter('_drv'),
doc="""The drive prefix (letter or UNC path), if any.""")
@@ -951,7 +863,6 @@
object. You can also instantiate a PosixPath or WindowsPath directly,
but cannot instantiate a WindowsPath on a POSIX system or vice versa.
"""
- _accessor = _normal_accessor
__slots__ = ()
def __new__(cls, *args, **kwargs):
@@ -970,17 +881,20 @@
return self._from_parsed_parts(self._drv, self._root, parts)
def __enter__(self):
+ # In previous versions of pathlib, __exit__() marked this path as
+ # closed; subsequent attempts to perform I/O would raise an IOError.
+ # This functionality was never documented, and had the effect of
+ # making Path objects mutable, contrary to PEP 428.
+ # In Python 3.9 __exit__() was made a no-op.
+ # In Python 3.11 __enter__() began emitting DeprecationWarning.
+ # In Python 3.13 __enter__() and __exit__() should be removed.
+ warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled "
+ "for removal in Python 3.13; Path objects as a context "
+ "manager is a no-op",
+ DeprecationWarning, stacklevel=2)
return self
def __exit__(self, t, v, tb):
- # https://bugs.python.org/issue39682
- # In previous versions of pathlib, this method marked this path as
- # closed; subsequent attempts to perform I/O would raise an IOError.
- # This functionality was never documented, and had the effect of
- # making Path objects mutable, contrary to PEP 428. In Python 3.9 the
- # _closed attribute was removed, and this method made a no-op.
- # This method and __enter__()/__exit__() should be deprecated and
- # removed in the future.
pass
# Public API
@@ -990,7 +904,7 @@
"""Return a new path pointing to the current working directory
(as returned by os.getcwd()).
"""
- return cls(cls._accessor.getcwd())
+ return cls(os.getcwd())
@classmethod
def home(cls):
@@ -1007,19 +921,22 @@
try:
other_st = other_path.stat()
except AttributeError:
- other_st = self._accessor.stat(other_path)
+ other_st = self.__class__(other_path).stat()
return os.path.samestat(st, other_st)
def iterdir(self):
"""Iterate over the files in this directory. Does not yield any
result for the special paths '.' and '..'.
"""
- for name in self._accessor.listdir(self):
- if name in {'.', '..'}:
- # Yielding a path object for these makes little sense
- continue
+ for name in os.listdir(self):
yield self._make_child_relpath(name)
+ def _scandir(self):
+ # bpo-24132: a future version of pathlib will support subclassing of
+ # pathlib.Path to customize how the filesystem is accessed. This
+ # includes scandir(), which is used to implement glob().
+ return os.scandir(self)
+
def glob(self, pattern):
"""Iterate over this subtree and yield all existing files (of any
kind, including directories) matching the given relative pattern.
@@ -1030,6 +947,8 @@
drv, root, pattern_parts = self._flavour.parse_parts((pattern,))
if drv or root:
raise NotImplementedError("Non-relative patterns are unsupported")
+ if pattern[-1] in (self._flavour.sep, self._flavour.altsep):
+ pattern_parts.append('')
selector = _make_selector(tuple(pattern_parts), self._flavour)
for p in selector.select_from(self):
yield p
@@ -1043,29 +962,26 @@
drv, root, pattern_parts = self._flavour.parse_parts((pattern,))
if drv or root:
raise NotImplementedError("Non-relative patterns are unsupported")
+ if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep):
+ pattern_parts.append('')
selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour)
for p in selector.select_from(self):
yield p
def absolute(self):
- """Return an absolute version of this path. This function works
- even if the path doesn't point to anything.
+ """Return an absolute version of this path by prepending the current
+ working directory. No normalization or symlink resolution is performed.
- No normalization is done, i.e. all '.' and '..' will be kept along.
Use resolve() to get the canonical path to a file.
"""
- # XXX untested yet!
if self.is_absolute():
return self
- # FIXME this must defer to the specific flavour (and, under Windows,
- # use nt._getfullpathname())
- return self._from_parts([self._accessor.getcwd()] + self._parts)
+ return self._from_parts([self.cwd()] + self._parts)
def resolve(self, strict=False):
"""
Make the path absolute, resolving all symlinks on the way and also
- normalizing it (for example turning slashes into backslashes under
- Windows).
+ normalizing it.
"""
def check_eloop(e):
@@ -1074,7 +990,7 @@
raise RuntimeError("Symlink loop from %r" % e.filename)
try:
- s = self._accessor.realpath(self, strict=strict)
+ s = os.path.realpath(self, strict=strict)
except OSError as e:
check_eloop(e)
raise
@@ -1094,19 +1010,28 @@
Return the result of the stat() system call on this path, like
os.stat() does.
"""
- return self._accessor.stat(self, follow_symlinks=follow_symlinks)
+ return os.stat(self, follow_symlinks=follow_symlinks)
def owner(self):
"""
Return the login name of the file owner.
"""
- return self._accessor.owner(self)
+ try:
+ import pwd
+ return pwd.getpwuid(self.stat().st_uid).pw_name
+ except ImportError:
+ raise NotImplementedError("Path.owner() is unsupported on this system")
def group(self):
"""
Return the group name of the file gid.
"""
- return self._accessor.group(self)
+
+ try:
+ import grp
+ return grp.getgrgid(self.stat().st_gid).gr_name
+ except ImportError:
+ raise NotImplementedError("Path.group() is unsupported on this system")
def open(self, mode='r', buffering=-1, encoding=None,
errors=None, newline=None):
@@ -1116,8 +1041,7 @@
"""
if "b" not in mode:
encoding = io.text_encoding(encoding)
- return self._accessor.open(self, mode, buffering, encoding, errors,
- newline)
+ return io.open(self, mode, buffering, encoding, errors, newline)
def read_bytes(self):
"""
@@ -1158,21 +1082,38 @@
"""
Return the path to which the symbolic link points.
"""
- path = self._accessor.readlink(self)
- return self._from_parts((path,))
+ if not hasattr(os, "readlink"):
+ raise NotImplementedError("os.readlink() not available on this system")
+ return self._from_parts((os.readlink(self),))
def touch(self, mode=0o666, exist_ok=True):
"""
Create this file with the given access mode, if it doesn't exist.
"""
- self._accessor.touch(self, mode, exist_ok)
+
+ if exist_ok:
+ # First try to bump modification time
+ # Implementation note: GNU touch uses the UTIME_NOW option of
+ # the utimensat() / futimens() functions.
+ try:
+ os.utime(self, None)
+ except OSError:
+ # Avoid exception chaining
+ pass
+ else:
+ return
+ flags = os.O_CREAT | os.O_WRONLY
+ if not exist_ok:
+ flags |= os.O_EXCL
+ fd = os.open(self, flags, mode)
+ os.close(fd)
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
"""
Create a new directory at this given path.
"""
try:
- self._accessor.mkdir(self, mode)
+ os.mkdir(self, mode)
except FileNotFoundError:
if not parents or self.parent == self:
raise
@@ -1188,7 +1129,7 @@
"""
Change the permissions of the path, like os.chmod().
"""
- self._accessor.chmod(self, mode, follow_symlinks=follow_symlinks)
+ os.chmod(self, mode, follow_symlinks=follow_symlinks)
def lchmod(self, mode):
"""
@@ -1203,7 +1144,7 @@
If the path is a directory, use rmdir() instead.
"""
try:
- self._accessor.unlink(self)
+ os.unlink(self)
except FileNotFoundError:
if not missing_ok:
raise
@@ -1212,7 +1153,7 @@
"""
Remove this directory. The directory must be empty.
"""
- self._accessor.rmdir(self)
+ os.rmdir(self)
def lstat(self):
"""
@@ -1231,7 +1172,7 @@
Returns the new Path instance pointing to the target path.
"""
- self._accessor.rename(self, target)
+ os.rename(self, target)
return self.__class__(target)
def replace(self, target):
@@ -1244,7 +1185,7 @@
Returns the new Path instance pointing to the target path.
"""
- self._accessor.replace(self, target)
+ os.replace(self, target)
return self.__class__(target)
def symlink_to(self, target, target_is_directory=False):
@@ -1252,7 +1193,9 @@
Make this path a symlink pointing to the target path.
Note the order of arguments (link, target) is the reverse of os.symlink.
"""
- self._accessor.symlink(target, self, target_is_directory)
+ if not hasattr(os, "symlink"):
+ raise NotImplementedError("os.symlink() not available on this system")
+ os.symlink(target, self, target_is_directory)
def hardlink_to(self, target):
"""
@@ -1260,7 +1203,9 @@
Note the order of arguments (self, target) is the reverse of os.link's.
"""
- self._accessor.link(target, self)
+ if not hasattr(os, "link"):
+ raise NotImplementedError("os.link() not available on this system")
+ os.link(target, self)
def link_to(self, target):
"""
@@ -1278,7 +1223,7 @@
"for removal in Python 3.12. "
"Use pathlib.Path.hardlink_to() instead.",
DeprecationWarning, stacklevel=2)
- self._accessor.link(self, target)
+ self.__class__(target).hardlink_to(self)
# Convenience functions for querying the stat results
@@ -1435,7 +1380,7 @@
"""
if (not (self._drv or self._root) and
self._parts and self._parts[0][:1] == '~'):
- homedir = self._accessor.expanduser(self._parts[0])
+ homedir = os.path.expanduser(self._parts[0])
if homedir[:1] == "~":
raise RuntimeError("Could not determine home directory.")
return self._from_parts([homedir] + self._parts[1:])
diff --git a/common/py3-stdlib/pdb.py b/common/py3-stdlib/pdb.py
index 3bdb2ac..d3824e1 100755
--- a/common/py3-stdlib/pdb.py
+++ b/common/py3-stdlib/pdb.py
@@ -80,9 +80,12 @@
import signal
import inspect
import tokenize
+import functools
import traceback
import linecache
+from typing import Union
+
class Restart(Exception):
"""Causes a debugger to be restarted for the debugged python program."""
@@ -119,6 +122,81 @@
return self
+class _ScriptTarget(str):
+ def __new__(cls, val):
+ # Mutate self to be the "real path".
+ res = super().__new__(cls, os.path.realpath(val))
+
+ # Store the original path for error reporting.
+ res.orig = val
+
+ return res
+
+ def check(self):
+ if not os.path.exists(self):
+ print('Error:', self.orig, 'does not exist')
+ sys.exit(1)
+
+ # Replace pdb's dir with script's dir in front of module search path.
+ sys.path[0] = os.path.dirname(self)
+
+ @property
+ def filename(self):
+ return self
+
+ @property
+ def namespace(self):
+ return dict(
+ __name__='__main__',
+ __file__=self,
+ __builtins__=__builtins__,
+ )
+
+ @property
+ def code(self):
+ with io.open_code(self) as fp:
+ return f"exec(compile({fp.read()!r}, {self!r}, 'exec'))"
+
+
+class _ModuleTarget(str):
+ def check(self):
+ try:
+ self._details
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+ @functools.cached_property
+ def _details(self):
+ import runpy
+ return runpy._get_module_details(self)
+
+ @property
+ def filename(self):
+ return self.code.co_filename
+
+ @property
+ def code(self):
+ name, spec, code = self._details
+ return code
+
+ @property
+ def _spec(self):
+ name, spec, code = self._details
+ return spec
+
+ @property
+ def namespace(self):
+ return dict(
+ __name__='__main__',
+ __file__=os.path.normcase(os.path.abspath(self.filename)),
+ __package__=self._spec.parent,
+ __loader__=self._spec.loader,
+ __spec__=self._spec,
+ __builtins__=__builtins__,
+ )
+
+
# Interaction prompt line will separate file and call info from code
# text using value of line_prefix string. A newline and arrow may
# be to your liking. You can set it once pdb is imported using the
@@ -157,12 +235,12 @@
self.rcLines = []
if readrc:
try:
- with open(os.path.expanduser('~/.pdbrc')) as rcFile:
+ with open(os.path.expanduser('~/.pdbrc'), encoding='utf-8') as rcFile:
self.rcLines.extend(rcFile)
except OSError:
pass
try:
- with open(".pdbrc") as rcFile:
+ with open(".pdbrc", encoding='utf-8') as rcFile:
self.rcLines.extend(rcFile)
except OSError:
pass
@@ -553,6 +631,12 @@
except:
self.error("Usage: commands [bnum]\n ...\n end")
return
+ try:
+ self.get_bpbynumber(bnum)
+ except ValueError as err:
+ self.error('cannot set commands: %s' % err)
+ return
+
self.commands_bnum = bnum
# Save old definitions for the case of a keyboard interrupt.
if bnum in self.commands:
@@ -1264,7 +1348,7 @@
filename = self.curframe.f_code.co_filename
breaklist = self.get_file_breaks(filename)
try:
- lines, lineno = inspect.getsourcelines(self.curframe)
+ lines, lineno = self._getsourcelines(self.curframe)
except OSError as err:
self.error(err)
return
@@ -1280,7 +1364,7 @@
except:
return
try:
- lines, lineno = inspect.getsourcelines(obj)
+ lines, lineno = self._getsourcelines(obj)
except (OSError, TypeError) as err:
self.error(err)
return
@@ -1538,49 +1622,36 @@
return fullname
return None
- def _runmodule(self, module_name):
- self._wait_for_mainpyfile = True
- self._user_requested_quit = False
- import runpy
- mod_name, mod_spec, code = runpy._get_module_details(module_name)
- self.mainpyfile = self.canonic(code.co_filename)
- import __main__
- __main__.__dict__.clear()
- __main__.__dict__.update({
- "__name__": "__main__",
- "__file__": self.mainpyfile,
- "__package__": mod_spec.parent,
- "__loader__": mod_spec.loader,
- "__spec__": mod_spec,
- "__builtins__": __builtins__,
- })
- self.run(code)
-
- def _runscript(self, filename):
- # The script has to run in __main__ namespace (or imports from
- # __main__ will break).
- #
- # So we clear up the __main__ and set several special variables
- # (this gets rid of pdb's globals and cleans old variables on restarts).
- import __main__
- __main__.__dict__.clear()
- __main__.__dict__.update({"__name__" : "__main__",
- "__file__" : filename,
- "__builtins__": __builtins__,
- })
-
- # When bdb sets tracing, a number of call and line events happens
+ def _run(self, target: Union[_ModuleTarget, _ScriptTarget]):
+ # When bdb sets tracing, a number of call and line events happen
# BEFORE debugger even reaches user's code (and the exact sequence of
- # events depends on python version). So we take special measures to
- # avoid stopping before we reach the main script (see user_line and
+ # events depends on python version). Take special measures to
+ # avoid stopping before reaching the main script (see user_line and
# user_call for details).
self._wait_for_mainpyfile = True
- self.mainpyfile = self.canonic(filename)
self._user_requested_quit = False
- with io.open_code(filename) as fp:
- statement = "exec(compile(%r, %r, 'exec'))" % \
- (fp.read(), self.mainpyfile)
- self.run(statement)
+
+ self.mainpyfile = self.canonic(target.filename)
+
+ # The target has to run in __main__ namespace (or imports from
+ # __main__ will break). Clear __main__ and replace with
+ # the target namespace.
+ import __main__
+ __main__.__dict__.clear()
+ __main__.__dict__.update(target.namespace)
+
+ self.run(target.code)
+
+
+ def _getsourcelines(self, obj):
+ # GH-103319
+ # inspect.getsourcelines() returns lineno = 0 for
+ # module-level frame which breaks our code print line number
+ # This method should be replaced by inspect.getsourcelines(obj)
+ # once this bug is fixed in inspect
+ lines, lineno = inspect.getsourcelines(obj)
+ lineno = max(1, lineno)
+ return lines, lineno
# Collect all command help into docstring, if not run with -OO
@@ -1669,6 +1740,7 @@
To let the script run up to a given line X in the debugged file, use
"-c 'until X'"."""
+
def main():
import getopt
@@ -1678,37 +1750,20 @@
print(_usage)
sys.exit(2)
- commands = []
- run_as_module = False
- for opt, optarg in opts:
- if opt in ['-h', '--help']:
- print(_usage)
- sys.exit()
- elif opt in ['-c', '--command']:
- commands.append(optarg)
- elif opt in ['-m']:
- run_as_module = True
+ if any(opt in ['-h', '--help'] for opt, optarg in opts):
+ print(_usage)
+ sys.exit()
- mainpyfile = args[0] # Get script filename
- if not run_as_module and not os.path.exists(mainpyfile):
- print('Error:', mainpyfile, 'does not exist')
- sys.exit(1)
+ commands = [optarg for opt, optarg in opts if opt in ['-c', '--command']]
- if run_as_module:
- import runpy
- try:
- runpy._get_module_details(mainpyfile)
- except Exception:
- traceback.print_exc()
- sys.exit(1)
+ module_indicated = any(opt in ['-m'] for opt, optarg in opts)
+ cls = _ModuleTarget if module_indicated else _ScriptTarget
+ target = cls(args[0])
+
+ target.check()
sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list
- if not run_as_module:
- mainpyfile = os.path.realpath(mainpyfile)
- # Replace pdb's dir with script's dir in front of module search path.
- sys.path[0] = os.path.dirname(mainpyfile)
-
# Note on saving/restoring sys.argv: it's a good idea when sys.argv was
# modified by the script being debugged. It's a bad idea when it was
# changed by the user from the command line. There is a "restart" command
@@ -1717,15 +1772,12 @@
pdb.rcLines.extend(commands)
while True:
try:
- if run_as_module:
- pdb._runmodule(mainpyfile)
- else:
- pdb._runscript(mainpyfile)
+ pdb._run(target)
if pdb._user_requested_quit:
break
print("The program finished and will be restarted")
except Restart:
- print("Restarting", mainpyfile, "with arguments:")
+ print("Restarting", target, "with arguments:")
print("\t" + " ".join(sys.argv[1:]))
except SystemExit:
# In most cases SystemExit does not warrant a post-mortem session.
@@ -1740,7 +1792,7 @@
print("Running 'cont' or 'step' will restart the program")
t = sys.exc_info()[2]
pdb.interaction(None, t)
- print("Post mortem debugger finished. The " + mainpyfile +
+ print("Post mortem debugger finished. The " + target +
" will be restarted")
diff --git a/common/py3-stdlib/pipes.py b/common/py3-stdlib/pipes.py
index 8cc74b0..61d63b4 100644
--- a/common/py3-stdlib/pipes.py
+++ b/common/py3-stdlib/pipes.py
@@ -60,10 +60,13 @@
import re
import os
import tempfile
+import warnings
# we import the quote function rather than the module for backward compat
# (quote used to be an undocumented but used function in pipes)
from shlex import quote
+warnings._deprecated(__name__, remove=(3, 13))
+
__all__ = ["Template"]
# Conversion step kinds
diff --git a/common/py3-stdlib/pkgutil.py b/common/py3-stdlib/pkgutil.py
index 8e010c7..56731de 100644
--- a/common/py3-stdlib/pkgutil.py
+++ b/common/py3-stdlib/pkgutil.py
@@ -413,6 +413,7 @@
The cache (or part of it) can be cleared manually if a
rescan of sys.path_hooks is necessary.
"""
+ path_item = os.fsdecode(path_item)
try:
importer = sys.path_importer_cache[path_item]
except KeyError:
@@ -510,10 +511,10 @@
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
- This will add to the package's __path__ all subdirectories of
- directories on sys.path named after the package. This is useful
- if one wants to distribute different parts of a single logical
- package as multiple directories.
+ For each directory on sys.path that has a subdirectory that
+ matches the package name, add the subdirectory to the package's
+ __path__. This is useful if one wants to distribute different
+ parts of a single logical package as multiple directories.
It also looks for *.pkg files beginning where * matches the name
argument. This feature is similar to *.pth files (see site.py),
diff --git a/common/py3-stdlib/platform.py b/common/py3-stdlib/platform.py
index 6a820c9..9b9d88b 100755
--- a/common/py3-stdlib/platform.py
+++ b/common/py3-stdlib/platform.py
@@ -5,7 +5,7 @@
If called from the command line, it prints the platform
information concatenated as single string to stdout. The output
- format is useable as part of a filename.
+ format is usable as part of a filename.
"""
# This module is maintained by Marc-Andre Lemburg <mal@egenix.com>.
@@ -116,7 +116,6 @@
import os
import re
import sys
-import subprocess
import functools
import itertools
@@ -169,7 +168,7 @@
Note that the function has intimate knowledge of how different
libc versions add symbols to the executable and thus is probably
- only useable for executables compiled using gcc.
+ only usable for executables compiled using gcc.
The file is read and scanned in chunks of chunksize bytes.
@@ -187,12 +186,15 @@
executable = sys.executable
+ if not executable:
+ # sys.executable is not set.
+ return lib, version
+
V = _comparable_version
- if hasattr(os.path, 'realpath'):
- # Python 2.2 introduced os.path.realpath(); it is used
- # here to work around problems with Cygwin not being
- # able to open symlinks for reading
- executable = os.path.realpath(executable)
+ # We use os.path.realpath()
+ # here to work around problems with Cygwin not being
+ # able to open symlinks for reading
+ executable = os.path.realpath(executable)
with open(executable, 'rb') as f:
binary = f.read(chunksize)
pos = 0
@@ -283,6 +285,7 @@
stdin=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
text=True,
+ encoding="locale",
shell=True)
except (OSError, subprocess.CalledProcessError) as why:
#print('Command %s failed: %s' % (cmd, why))
@@ -609,7 +612,10 @@
# XXX Others too ?
return default
- import subprocess
+ try:
+ import subprocess
+ except ImportError:
+ return default
target = _follow_symlinks(target)
# "file" output is locale dependent: force the usage of the C locale
# to get deterministic behavior.
@@ -749,10 +755,15 @@
Fall back to `uname -p`
"""
try:
+ import subprocess
+ except ImportError:
+ return None
+ try:
return subprocess.check_output(
['uname', '-p'],
stderr=subprocess.DEVNULL,
text=True,
+ encoding="utf8",
).strip()
except (OSError, subprocess.CalledProcessError):
pass
diff --git a/common/py3-stdlib/plistlib.py b/common/py3-stdlib/plistlib.py
index d6c997e..53e718f 100644
--- a/common/py3-stdlib/plistlib.py
+++ b/common/py3-stdlib/plistlib.py
@@ -187,8 +187,8 @@
return self.root
def handle_entity_decl(self, entity_name, is_parameter_entity, value, base, system_id, public_id, notation_name):
- # Reject plist files with entity declarations to avoid XML vulnerabilies in expat.
- # Regular plist files don't contain those declerations, and Apple's plutil tool does not
+ # Reject plist files with entity declarations to avoid XML vulnerabilities in expat.
+ # Regular plist files don't contain those declarations, and Apple's plutil tool does not
# accept them either.
raise InvalidFileException("XML entity declarations are not supported in plist files")
diff --git a/common/py3-stdlib/posixpath.py b/common/py3-stdlib/posixpath.py
index e550b47..5b4d78b 100644
--- a/common/py3-stdlib/posixpath.py
+++ b/common/py3-stdlib/posixpath.py
@@ -242,7 +242,11 @@
i = len(path)
if i == 1:
if 'HOME' not in os.environ:
- import pwd
+ try:
+ import pwd
+ except ImportError:
+ # pwd module unavailable, return path unchanged
+ return path
try:
userhome = pwd.getpwuid(os.getuid()).pw_dir
except KeyError:
@@ -252,7 +256,11 @@
else:
userhome = os.environ['HOME']
else:
- import pwd
+ try:
+ import pwd
+ except ImportError:
+ # pwd module unavailable, return path unchanged
+ return path
name = path[1:i]
if isinstance(name, bytes):
name = str(name, 'ASCII')
@@ -335,43 +343,55 @@
# It should be understood that this may change the meaning of the path
# if it contains symbolic links!
-def normpath(path):
- """Normalize path, eliminating double slashes, etc."""
- path = os.fspath(path)
- if isinstance(path, bytes):
- sep = b'/'
- empty = b''
- dot = b'.'
- dotdot = b'..'
- else:
- sep = '/'
- empty = ''
- dot = '.'
- dotdot = '..'
- if path == empty:
- return dot
- initial_slashes = path.startswith(sep)
- # POSIX allows one or two initial slashes, but treats three or more
- # as single slash.
- # (see https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
- if (initial_slashes and
- path.startswith(sep*2) and not path.startswith(sep*3)):
- initial_slashes = 2
- comps = path.split(sep)
- new_comps = []
- for comp in comps:
- if comp in (empty, dot):
- continue
- if (comp != dotdot or (not initial_slashes and not new_comps) or
- (new_comps and new_comps[-1] == dotdot)):
- new_comps.append(comp)
- elif new_comps:
- new_comps.pop()
- comps = new_comps
- path = sep.join(comps)
- if initial_slashes:
- path = sep*initial_slashes + path
- return path or dot
+try:
+ from posix import _path_normpath
+
+except ImportError:
+ def normpath(path):
+ """Normalize path, eliminating double slashes, etc."""
+ path = os.fspath(path)
+ if isinstance(path, bytes):
+ sep = b'/'
+ empty = b''
+ dot = b'.'
+ dotdot = b'..'
+ else:
+ sep = '/'
+ empty = ''
+ dot = '.'
+ dotdot = '..'
+ if path == empty:
+ return dot
+ initial_slashes = path.startswith(sep)
+ # POSIX allows one or two initial slashes, but treats three or more
+ # as single slash.
+ # (see https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
+ if (initial_slashes and
+ path.startswith(sep*2) and not path.startswith(sep*3)):
+ initial_slashes = 2
+ comps = path.split(sep)
+ new_comps = []
+ for comp in comps:
+ if comp in (empty, dot):
+ continue
+ if (comp != dotdot or (not initial_slashes and not new_comps) or
+ (new_comps and new_comps[-1] == dotdot)):
+ new_comps.append(comp)
+ elif new_comps:
+ new_comps.pop()
+ comps = new_comps
+ path = sep.join(comps)
+ if initial_slashes:
+ path = sep*initial_slashes + path
+ return path or dot
+
+else:
+ def normpath(path):
+ """Normalize path, eliminating double slashes, etc."""
+ path = os.fspath(path)
+ if isinstance(path, bytes):
+ return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
+ return _path_normpath(path) or "."
def abspath(path):
diff --git a/common/py3-stdlib/pprint.py b/common/py3-stdlib/pprint.py
index d91421f..575688d 100644
--- a/common/py3-stdlib/pprint.py
+++ b/common/py3-stdlib/pprint.py
@@ -149,8 +149,9 @@
self._underscore_numbers = underscore_numbers
def pprint(self, object):
- self._format(object, self._stream, 0, 0, {}, 0)
- self._stream.write("\n")
+ if self._stream is not None:
+ self._format(object, self._stream, 0, 0, {}, 0)
+ self._stream.write("\n")
def pformat(self, object):
sio = _StringIO()
diff --git a/common/py3-stdlib/profile.py b/common/py3-stdlib/profile.py
index d8599fb..90c4e4c 100755
--- a/common/py3-stdlib/profile.py
+++ b/common/py3-stdlib/profile.py
@@ -24,6 +24,7 @@
# governing permissions and limitations under the License.
+import io
import sys
import time
import marshal
@@ -587,7 +588,7 @@
else:
progname = args[0]
sys.path.insert(0, os.path.dirname(progname))
- with open(progname, 'rb') as fp:
+ with io.open_code(progname) as fp:
code = compile(fp.read(), progname, 'exec')
globs = {
'__file__': progname,
diff --git a/common/py3-stdlib/pstats.py b/common/py3-stdlib/pstats.py
index ac88c04..8040831 100644
--- a/common/py3-stdlib/pstats.py
+++ b/common/py3-stdlib/pstats.py
@@ -26,14 +26,15 @@
import marshal
import re
-from enum import Enum
+from enum import StrEnum, _simple_enum
from functools import cmp_to_key
from dataclasses import dataclass
from typing import Dict
__all__ = ["Stats", "SortKey", "FunctionProfile", "StatsProfile"]
-class SortKey(str, Enum):
+@_simple_enum(StrEnum)
+class SortKey:
CALLS = 'calls', 'ncalls'
CUMULATIVE = 'cumulative', 'cumtime'
FILENAME = 'filename', 'module'
@@ -521,7 +522,7 @@
class TupleComp:
"""This class provides a generic function for comparing any two tuples.
Each instance records a list of tuple-indices (from most significant
- to least significant), and sort direction (ascending or decending) for
+ to least significant), and sort direction (ascending or descending) for
each tuple-index. The compare functions can then be used as the function
argument to the system sort() function when a list of tuples need to be
sorted in the instances order."""
diff --git a/common/py3-stdlib/pty.py b/common/py3-stdlib/pty.py
index 8d8ce40..fefb63a 100644
--- a/common/py3-stdlib/pty.py
+++ b/common/py3-stdlib/pty.py
@@ -121,12 +121,6 @@
# Parent and child process.
return pid, master_fd
-def _writen(fd, data):
- """Write all the data to a descriptor."""
- while data:
- n = os.write(fd, data)
- data = data[n:]
-
def _read(fd):
"""Default read function."""
return os.read(fd, 1024)
@@ -136,9 +130,42 @@
Copies
pty master -> standard output (master_read)
standard input -> pty master (stdin_read)"""
- fds = [master_fd, STDIN_FILENO]
- while fds:
- rfds, _wfds, _xfds = select(fds, [], [])
+ if os.get_blocking(master_fd):
+ # If we write more than tty/ndisc is willing to buffer, we may block
+ # indefinitely. So we set master_fd to non-blocking temporarily during
+ # the copy operation.
+ os.set_blocking(master_fd, False)
+ try:
+ _copy(master_fd, master_read=master_read, stdin_read=stdin_read)
+ finally:
+ # restore blocking mode for backwards compatibility
+ os.set_blocking(master_fd, True)
+ return
+ high_waterlevel = 4096
+ stdin_avail = master_fd != STDIN_FILENO
+ stdout_avail = master_fd != STDOUT_FILENO
+ i_buf = b''
+ o_buf = b''
+ while 1:
+ rfds = []
+ wfds = []
+ if stdin_avail and len(i_buf) < high_waterlevel:
+ rfds.append(STDIN_FILENO)
+ if stdout_avail and len(o_buf) < high_waterlevel:
+ rfds.append(master_fd)
+ if stdout_avail and len(o_buf) > 0:
+ wfds.append(STDOUT_FILENO)
+ if len(i_buf) > 0:
+ wfds.append(master_fd)
+
+ rfds, wfds, _xfds = select(rfds, wfds, [])
+
+ if STDOUT_FILENO in wfds:
+ try:
+ n = os.write(STDOUT_FILENO, o_buf)
+ o_buf = o_buf[n:]
+ except OSError:
+ stdout_avail = False
if master_fd in rfds:
# Some OSes signal EOF by returning an empty byte string,
@@ -150,15 +177,18 @@
if not data: # Reached EOF.
return # Assume the child process has exited and is
# unreachable, so we clean up.
- else:
- os.write(STDOUT_FILENO, data)
+ o_buf += data
- if STDIN_FILENO in rfds:
+ if master_fd in wfds:
+ n = os.write(master_fd, i_buf)
+ i_buf = i_buf[n:]
+
+ if stdin_avail and STDIN_FILENO in rfds:
data = stdin_read(STDIN_FILENO)
if not data:
- fds.remove(STDIN_FILENO)
+ stdin_avail = False
else:
- _writen(master_fd, data)
+ i_buf += data
def spawn(argv, master_read=_read, stdin_read=_read):
"""Create a spawned process."""
diff --git a/common/py3-stdlib/pydoc.py b/common/py3-stdlib/pydoc.py
index e00ba41..088a3ba 100755
--- a/common/py3-stdlib/pydoc.py
+++ b/common/py3-stdlib/pydoc.py
@@ -54,6 +54,7 @@
# the current directory is changed with os.chdir(), an incorrect
# path will be displayed.
+import __future__
import builtins
import importlib._bootstrap
import importlib._bootstrap_external
@@ -69,7 +70,6 @@
import sysconfig
import time
import tokenize
-import types
import urllib.parse
import warnings
from collections import deque
@@ -91,16 +91,13 @@
normdirs.append(normdir)
return dirs
-def _isclass(object):
- return inspect.isclass(object) and not isinstance(object, types.GenericAlias)
-
def _findclass(func):
cls = sys.modules.get(func.__module__)
if cls is None:
return None
for name in func.__qualname__.split('.')[:-1]:
cls = getattr(cls, name)
- if not _isclass(cls):
+ if not inspect.isclass(cls):
return None
return cls
@@ -108,7 +105,7 @@
if inspect.ismethod(obj):
name = obj.__func__.__name__
self = obj.__self__
- if (_isclass(self) and
+ if (inspect.isclass(self) and
getattr(getattr(self, name, None), '__func__') is obj.__func__):
# classmethod
cls = self
@@ -122,7 +119,7 @@
elif inspect.isbuiltin(obj):
name = obj.__name__
self = obj.__self__
- if (_isclass(self) and
+ if (inspect.isclass(self) and
self.__qualname__ + '.' + name == obj.__qualname__):
# classmethod
cls = self
@@ -209,7 +206,7 @@
def isdata(object):
"""Check if an object is of a type that probably means it's data."""
- return not (inspect.ismodule(object) or _isclass(object) or
+ return not (inspect.ismodule(object) or inspect.isclass(object) or
inspect.isroutine(object) or inspect.isframe(object) or
inspect.istraceback(object) or inspect.iscode(object))
@@ -274,6 +271,8 @@
no.append(x)
return yes, no
+_future_feature_names = set(__future__.all_feature_names)
+
def visiblename(name, all=None, obj=None):
"""Decide whether to show documentation on a variable."""
# Certain special names are redundant or internal.
@@ -288,6 +287,10 @@
# Namedtuples have public fields and methods with a single leading underscore
if name.startswith('_') and hasattr(obj, '_fields'):
return True
+ # Ignore __future__ imports.
+ if obj is not __future__ and name in _future_feature_names:
+ if isinstance(getattr(obj, name, None), __future__._Feature):
+ return False
if all is not None:
# only document that which the programmer exported in __all__
return name in all
@@ -474,7 +477,7 @@
# by lacking a __name__ attribute) and an instance.
try:
if inspect.ismodule(object): return self.docmodule(*args)
- if _isclass(object): return self.docclass(*args)
+ if inspect.isclass(object): return self.docclass(*args)
if inspect.isroutine(object): return self.docroutine(*args)
except AttributeError:
pass
@@ -546,7 +549,7 @@
# needed to make any special characters, so show a raw string.
return 'r' + testrepr[0] + self.escape(test) + testrepr[0]
return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)',
- r'<font color="#c040c0">\1</font>',
+ r'<span class="repr">\1</span>',
self.escape(testrepr))
repr_str = repr_string
@@ -571,49 +574,48 @@
def page(self, title, contents):
"""Format an HTML page."""
return '''\
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Python: %s</title>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-</head><body bgcolor="#f0f0f8">
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Python: %s</title>
+</head><body>
%s
</body></html>''' % (title, contents)
- def heading(self, title, fgcol, bgcol, extras=''):
+ def heading(self, title, extras=''):
"""Format a page heading."""
return '''
-<table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="heading">
-<tr bgcolor="%s">
-<td valign=bottom> <br>
-<font color="%s" face="helvetica, arial"> <br>%s</font></td
-><td align=right valign=bottom
-><font color="%s" face="helvetica, arial">%s</font></td></tr></table>
- ''' % (bgcol, fgcol, title, fgcol, extras or ' ')
+<table class="heading">
+<tr class="heading-text decor">
+<td class="title"> <br>%s</td>
+<td class="extra">%s</td></tr></table>
+ ''' % (title, extras or ' ')
- def section(self, title, fgcol, bgcol, contents, width=6,
+ def section(self, title, cls, contents, width=6,
prelude='', marginalia=None, gap=' '):
"""Format a section with a heading."""
if marginalia is None:
- marginalia = '<tt>' + ' ' * width + '</tt>'
+ marginalia = '<span class="code">' + ' ' * width + '</span>'
result = '''<p>
-<table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="section">
-<tr bgcolor="%s">
-<td colspan=3 valign=bottom> <br>
-<font color="%s" face="helvetica, arial">%s</font></td></tr>
- ''' % (bgcol, fgcol, title)
+<table class="section">
+<tr class="decor %s-decor heading-text">
+<td class="section-title" colspan=3> <br>%s</td></tr>
+ ''' % (cls, title)
if prelude:
result = result + '''
-<tr bgcolor="%s"><td rowspan=2>%s</td>
-<td colspan=2>%s</td></tr>
-<tr><td>%s</td>''' % (bgcol, marginalia, prelude, gap)
+<tr><td class="decor %s-decor" rowspan=2>%s</td>
+<td class="decor %s-decor" colspan=2>%s</td></tr>
+<tr><td>%s</td>''' % (cls, marginalia, cls, prelude, gap)
else:
result = result + '''
-<tr><td bgcolor="%s">%s</td><td>%s</td>''' % (bgcol, marginalia, gap)
+<tr><td class="decor %s-decor">%s</td><td>%s</td>''' % (cls, marginalia, gap)
- return result + '\n<td width="100%%">%s</td></tr></table>' % contents
+ return result + '\n<td class="singlecolumn">%s</td></tr></table>' % contents
def bigsection(self, title, *args):
"""Format a section with a big heading."""
- title = '<big><strong>%s</strong></big>' % title
+ title = '<strong class="bigsection">%s</strong>' % title
return self.section(title, *args)
def preformat(self, text):
@@ -622,19 +624,19 @@
return replace(text, '\n\n', '\n \n', '\n\n', '\n \n',
' ', ' ', '\n', '<br>\n')
- def multicolumn(self, list, format, cols=4):
+ def multicolumn(self, list, format):
"""Format a list of items into a multi-column list."""
result = ''
- rows = (len(list)+cols-1)//cols
- for col in range(cols):
- result = result + '<td width="%d%%" valign=top>' % (100//cols)
+ rows = (len(list) + 3) // 4
+ for col in range(4):
+ result = result + '<td class="multicolumn">'
for i in range(rows*col, rows*col+rows):
if i < len(list):
result = result + format(list[i]) + '<br>\n'
result = result + '</td>'
- return '<table width="100%%" summary="list"><tr>%s</tr></table>' % result
+ return '<table><tr>%s</tr></table>' % result
- def grey(self, text): return '<font color="#909090">%s</font>' % text
+ def grey(self, text): return '<span class="grey">%s</span>' % text
def namelink(self, name, *dicts):
"""Make a link for an identifier, given name-to-URL mappings."""
@@ -695,10 +697,10 @@
url = escape(all).replace('"', '"')
results.append('<a href="%s">%s</a>' % (url, url))
elif rfc:
- url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
+ url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif pep:
- url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep)
+ url = 'https://peps.python.org/pep-%04d/' % int(pep)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif selfdot:
# Create a link for methods like 'self.method(...)'
@@ -723,14 +725,14 @@
for entry in tree:
if type(entry) is type(()):
c, bases = entry
- result = result + '<dt><font face="helvetica, arial">'
+ result = result + '<dt class="heading-text">'
result = result + self.classlink(c, modname)
if bases and bases != (parent,):
parents = []
for base in bases:
parents.append(self.classlink(base, modname))
result = result + '(' + ', '.join(parents) + ')'
- result = result + '\n</font></dt>'
+ result = result + '\n</dt>'
elif type(entry) is type([]):
result = result + '<dd>\n%s</dd>\n' % self.formattree(
entry, modname, c)
@@ -747,10 +749,10 @@
links = []
for i in range(len(parts)-1):
links.append(
- '<a href="%s.html"><font color="#ffffff">%s</font></a>' %
+ '<a href="%s.html" class="white">%s</a>' %
('.'.join(parts[:i+1]), parts[i]))
linkedname = '.'.join(links + parts[-1:])
- head = '<big><big><strong>%s</strong></big></big>' % linkedname
+ head = '<strong class="title">%s</strong>' % linkedname
try:
path = inspect.getabsfile(object)
url = urllib.parse.quote(path)
@@ -772,14 +774,12 @@
docloc = '<br><a href="%(docloc)s">Module Reference</a>' % locals()
else:
docloc = ''
- result = self.heading(
- head, '#ffffff', '#7799ee',
- '<a href=".">index</a><br>' + filelink + docloc)
+ result = self.heading(head, '<a href=".">index</a><br>' + filelink + docloc)
modules = inspect.getmembers(object, inspect.ismodule)
classes, cdict = [], {}
- for key, value in inspect.getmembers(object, _isclass):
+ for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None or
(inspect.getmodule(value) or object) is object):
@@ -809,7 +809,7 @@
data.append((key, value))
doc = self.markup(getdoc(object), self.preformat, fdict, cdict)
- doc = doc and '<tt>%s</tt>' % doc
+ doc = doc and '<span class="code">%s</span>' % doc
result = result + '<p>%s</p>\n' % doc
if hasattr(object, '__path__'):
@@ -819,12 +819,12 @@
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
result = result + self.bigsection(
- 'Package Contents', '#ffffff', '#aa55cc', contents)
+ 'Package Contents', 'pkg-content', contents)
elif modules:
contents = self.multicolumn(
modules, lambda t: self.modulelink(t[1]))
result = result + self.bigsection(
- 'Modules', '#ffffff', '#aa55cc', contents)
+ 'Modules', 'pkg-content', contents)
if classes:
classlist = [value for (key, value) in classes]
@@ -833,27 +833,25 @@
for key, value in classes:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
- 'Classes', '#ffffff', '#ee77aa', ' '.join(contents))
+ 'Classes', 'index', ' '.join(contents))
if funcs:
contents = []
for key, value in funcs:
contents.append(self.document(value, key, name, fdict, cdict))
result = result + self.bigsection(
- 'Functions', '#ffffff', '#eeaa77', ' '.join(contents))
+ 'Functions', 'functions', ' '.join(contents))
if data:
contents = []
for key, value in data:
contents.append(self.document(value, key))
result = result + self.bigsection(
- 'Data', '#ffffff', '#55aa55', '<br>\n'.join(contents))
+ 'Data', 'data', '<br>\n'.join(contents))
if hasattr(object, '__author__'):
contents = self.markup(str(object.__author__), self.preformat)
- result = result + self.bigsection(
- 'Author', '#ffffff', '#7799ee', contents)
+ result = result + self.bigsection('Author', 'author', contents)
if hasattr(object, '__credits__'):
contents = self.markup(str(object.__credits__), self.preformat)
- result = result + self.bigsection(
- 'Credits', '#ffffff', '#7799ee', contents)
+ result = result + self.bigsection('Credits', 'credits', contents)
return result
@@ -927,7 +925,7 @@
else:
doc = self.markup(getdoc(value), self.preformat,
funcs, classes, mdict)
- doc = '<dd><tt>%s</tt>' % doc
+ doc = '<dd><span class="code">%s</span>' % doc
push('<dl><dt>%s%s</dl>\n' % (base, doc))
push('\n')
return attrs
@@ -1015,9 +1013,9 @@
if decl:
doc = decl + (doc or '')
doc = self.markup(doc, self.preformat, funcs, classes, mdict)
- doc = doc and '<tt>%s<br> </tt>' % doc
+ doc = doc and '<span class="code">%s<br> </span>' % doc
- return self.section(title, '#000000', '#ffc8d8', contents, 3, doc)
+ return self.section(title, 'title', contents, 3, doc)
def formatvalue(self, object):
"""Format an argument default value as text."""
@@ -1078,14 +1076,14 @@
argspec = '(...)'
decl = asyncqualifier + title + self.escape(argspec) + (note and
- self.grey('<font face="helvetica, arial">%s</font>' % note))
+ self.grey('<span class="heading-text">%s</span>' % note))
if skipdocs:
return '<dl><dt>%s</dt></dl>\n' % decl
else:
doc = self.markup(
getdoc(object), self.preformat, funcs, classes, methods)
- doc = doc and '<dd><tt>%s</tt></dd>' % doc
+ doc = doc and '<dd><span class="code">%s</span></dd>' % doc
return '<dl><dt>%s</dt>%s</dl>\n' % (decl, doc)
def docdata(self, object, name=None, mod=None, cl=None):
@@ -1097,7 +1095,7 @@
push('<dl><dt><strong>%s</strong></dt>\n' % name)
doc = self.markup(getdoc(object), self.preformat)
if doc:
- push('<dd><tt>%s</tt></dd>\n' % doc)
+ push('<dd><span class="code">%s</span></dd>\n' % doc)
push('</dl>\n')
return ''.join(results)
@@ -1122,7 +1120,7 @@
modpkgs.sort()
contents = self.multicolumn(modpkgs, self.modpkglink)
- return self.bigsection(dir, '#ffffff', '#ee77aa', contents)
+ return self.bigsection(dir, 'index', contents)
# -------------------------------------------- text documentation generator
@@ -1221,7 +1219,7 @@
result = result + self.section('DESCRIPTION', desc)
classes = []
- for key, value in inspect.getmembers(object, _isclass):
+ for key, value in inspect.getmembers(object, inspect.isclass):
# if __all__ exists, believe it. Otherwise use old heuristic.
if (all is not None
or (inspect.getmodule(value) or object) is object):
@@ -1565,6 +1563,8 @@
return plainpager
if not sys.stdin.isatty() or not sys.stdout.isatty():
return plainpager
+ if sys.platform == "emscripten":
+ return plainpager
use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
if use_pager:
if sys.platform == 'win32': # pipes completely broken in Windows
@@ -1703,7 +1703,7 @@
return 'member descriptor %s.%s.%s' % (
thing.__objclass__.__module__, thing.__objclass__.__name__,
thing.__name__)
- if _isclass(thing):
+ if inspect.isclass(thing):
return 'class ' + thing.__name__
if inspect.isfunction(thing):
return 'function ' + thing.__name__
@@ -1764,7 +1764,7 @@
desc += ' in module ' + module.__name__
if not (inspect.ismodule(object) or
- _isclass(object) or
+ inspect.isclass(object) or
inspect.isroutine(object) or
inspect.isdatadescriptor(object) or
_getdoc(object)):
@@ -1780,24 +1780,18 @@
def doc(thing, title='Python Library Documentation: %s', forceload=0,
output=None):
"""Display text documentation, given an object or a path to an object."""
- try:
- if output is None:
- pager(render_doc(thing, title, forceload))
- else:
- output.write(render_doc(thing, title, forceload, plaintext))
- except (ImportError, ErrorDuringImport) as value:
- print(value)
+ if output is None:
+ pager(render_doc(thing, title, forceload))
+ else:
+ output.write(render_doc(thing, title, forceload, plaintext))
def writedoc(thing, forceload=0):
"""Write HTML documentation to a file in the current directory."""
- try:
- object, name = resolve(thing, forceload)
- page = html.page(describe(object), html.document(object, name))
- with open(name + '.html', 'w', encoding='utf-8') as file:
- file.write(page)
- print('wrote', name + '.html')
- except (ImportError, ErrorDuringImport) as value:
- print(value)
+ object, name = resolve(thing, forceload)
+ page = html.page(describe(object), html.document(object, name))
+ with open(name + '.html', 'w', encoding='utf-8') as file:
+ file.write(page)
+ print('wrote', name + '.html')
def writedocs(dir, pkgpath='', done=None):
"""Write out HTML documentation for all modules in a directory tree."""
@@ -1893,6 +1887,7 @@
if topic not in topics:
topics = topics + ' ' + topic
symbols[symbol] = topics
+ del topic, symbols_, symbol, topics
topics = {
'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS '
@@ -2003,7 +1998,10 @@
_GoInteractive = object()
def __call__(self, request=_GoInteractive):
if request is not self._GoInteractive:
- self.help(request)
+ try:
+ self.help(request)
+ except ImportError as e:
+ self.output.write(f'{e}\n')
else:
self.intro()
self.interact()
@@ -2456,10 +2454,12 @@
'<link rel="stylesheet" type="text/css" href="%s">' %
css_path)
return '''\
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Pydoc: %s</title>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-%s</head><body bgcolor="#f0f0f8">%s<div style="clear:both;padding-top:.5em;">%s</div>
+<!DOCTYPE>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Pydoc: %s</title>
+%s</head><body>%s<div style="clear:both;padding-top:.5em;">%s</div>
</body></html>''' % (title, css_link, html_navbar(), contents)
@@ -2499,22 +2499,21 @@
return '<a href="%s.html">%s</a>' % (name, name)
heading = html.heading(
- '<big><big><strong>Index of Modules</strong></big></big>',
- '#ffffff', '#7799ee')
+ '<strong class="title">Index of Modules</strong>'
+ )
names = [name for name in sys.builtin_module_names
if name != '__main__']
contents = html.multicolumn(names, bltinlink)
contents = [heading, '<p>' + html.bigsection(
- 'Built-in Modules', '#ffffff', '#ee77aa', contents)]
+ 'Built-in Modules', 'index', contents)]
seen = {}
for dir in sys.path:
contents.append(html.index(dir, seen))
contents.append(
- '<p align=right><font color="#909090" face="helvetica,'
- 'arial"><strong>pydoc</strong> by Ka-Ping Yee'
- '<ping@lfw.org></font>')
+ '<p align=right class="heading-text grey"><strong>pydoc</strong> by Ka-Ping Yee'
+ '<ping@lfw.org></p>')
return 'Index of Modules', ''.join(contents)
def html_search(key):
@@ -2539,12 +2538,12 @@
results = []
heading = html.heading(
- '<big><big><strong>Search Results</strong></big></big>',
- '#ffffff', '#7799ee')
+ '<strong class="title">Search Results</strong>',
+ )
for name, desc in search_result:
results.append(bltinlink(name) + desc)
contents = heading + html.bigsection(
- 'key = %s' % key, '#ffffff', '#ee77aa', '<br>'.join(results))
+ 'key = %s' % key, 'index', '<br>'.join(results))
return 'Search Results', contents
def html_topics():
@@ -2554,20 +2553,20 @@
return '<a href="topic?key=%s">%s</a>' % (name, name)
heading = html.heading(
- '<big><big><strong>INDEX</strong></big></big>',
- '#ffffff', '#7799ee')
+ '<strong class="title">INDEX</strong>',
+ )
names = sorted(Helper.topics.keys())
contents = html.multicolumn(names, bltinlink)
contents = heading + html.bigsection(
- 'Topics', '#ffffff', '#ee77aa', contents)
+ 'Topics', 'index', contents)
return 'Topics', contents
def html_keywords():
"""Index of keywords."""
heading = html.heading(
- '<big><big><strong>INDEX</strong></big></big>',
- '#ffffff', '#7799ee')
+ '<strong class="title">INDEX</strong>',
+ )
names = sorted(Helper.keywords.keys())
def bltinlink(name):
@@ -2575,7 +2574,7 @@
contents = html.multicolumn(names, bltinlink)
contents = heading + html.bigsection(
- 'Keywords', '#ffffff', '#ee77aa', contents)
+ 'Keywords', 'index', contents)
return 'Keywords', contents
def html_topicpage(topic):
@@ -2588,10 +2587,10 @@
else:
title = 'TOPIC'
heading = html.heading(
- '<big><big><strong>%s</strong></big></big>' % title,
- '#ffffff', '#7799ee')
+ '<strong class="title">%s</strong>' % title,
+ )
contents = '<pre>%s</pre>' % html.markup(contents)
- contents = html.bigsection(topic , '#ffffff','#ee77aa', contents)
+ contents = html.bigsection(topic , 'index', contents)
if xrefs:
xrefs = sorted(xrefs.split())
@@ -2599,8 +2598,7 @@
return '<a href="topic?key=%s">%s</a>' % (name, name)
xrefs = html.multicolumn(xrefs, bltinlink)
- xrefs = html.section('Related help topics: ',
- '#ffffff', '#ee77aa', xrefs)
+ xrefs = html.section('Related help topics: ', 'index', xrefs)
return ('%s %s' % (title, topic),
''.join((heading, contents, xrefs)))
@@ -2614,12 +2612,11 @@
def html_error(url, exc):
heading = html.heading(
- '<big><big><strong>Error</strong></big></big>',
- '#ffffff', '#7799ee')
+ '<strong class="title">Error</strong>',
+ )
contents = '<br>'.join(html.escape(line) for line in
format_exception_only(type(exc), exc))
- contents = heading + html.bigsection(url, '#ffffff', '#bb0000',
- contents)
+ contents = heading + html.bigsection(url, 'error', contents)
return "Error - %s" % url, contents
def get_html_page(url):
@@ -2791,7 +2788,7 @@
for arg in args:
if ispath(arg) and not os.path.exists(arg):
print('file %r does not exist' % arg)
- break
+ sys.exit(1)
try:
if ispath(arg) and os.path.isfile(arg):
arg = importfile(arg)
@@ -2802,8 +2799,9 @@
writedoc(arg)
else:
help.help(arg)
- except ErrorDuringImport as value:
+ except (ImportError, ErrorDuringImport) as value:
print(value)
+ sys.exit(1)
except (getopt.error, BadUsage):
cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0]
diff --git a/common/py3-stdlib/pydoc_data/topics.py b/common/py3-stdlib/pydoc_data/topics.py
index 1637f38..8093041 100644
--- a/common/py3-stdlib/pydoc_data/topics.py
+++ b/common/py3-stdlib/pydoc_data/topics.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Autogenerated by Sphinx on Tue Apr 4 22:56:51 2023
+# Autogenerated by Sphinx on Tue Jun 6 23:00:07 2023
topics = {'assert': 'The "assert" statement\n'
'**********************\n'
'\n'
@@ -1000,14 +1000,12 @@
' "A.__dict__[\'x\'].__get__(None, A)".\n'
'\n'
'Super Binding\n'
- ' If "a" is an instance of "super", then the binding '
- '"super(B,\n'
- ' obj).m()" searches "obj.__class__.__mro__" for the '
- 'base class "A"\n'
- ' immediately following "B" and then invokes the '
- 'descriptor with the\n'
- ' call: "A.__dict__[\'m\'].__get__(obj, '
- 'obj.__class__)".\n'
+ ' A dotted lookup such as "super(A, a).x" searches\n'
+ ' "a.__class__.__mro__" for a base class "B" following '
+ '"A" and then\n'
+ ' returns "B.__dict__[\'x\'].__get__(a, A)". If not a '
+ 'descriptor, "x"\n'
+ ' is returned unchanged.\n'
'\n'
'For instance bindings, the precedence of descriptor '
'invocation depends\n'
@@ -1586,25 +1584,24 @@
'parameter name, the first slot is used, and so on). If the slot '
'is\n'
'already filled, a "TypeError" exception is raised. Otherwise, the\n'
- 'value of the argument is placed in the slot, filling it (even if '
- 'the\n'
- 'expression is "None", it fills the slot). When all arguments have\n'
- 'been processed, the slots that are still unfilled are filled with '
- 'the\n'
- 'corresponding default value from the function definition. '
- '(Default\n'
- 'values are calculated, once, when the function is defined; thus, a\n'
- 'mutable object such as a list or dictionary used as default value '
- 'will\n'
- 'be shared by all calls that don’t specify an argument value for '
- 'the\n'
- 'corresponding slot; this should usually be avoided.) If there are '
- 'any\n'
- 'unfilled slots for which no default value is specified, a '
- '"TypeError"\n'
- 'exception is raised. Otherwise, the list of filled slots is used '
- 'as\n'
- 'the argument list for the call.\n'
+ 'argument is placed in the slot, filling it (even if the expression '
+ 'is\n'
+ '"None", it fills the slot). When all arguments have been '
+ 'processed,\n'
+ 'the slots that are still unfilled are filled with the '
+ 'corresponding\n'
+ 'default value from the function definition. (Default values are\n'
+ 'calculated, once, when the function is defined; thus, a mutable '
+ 'object\n'
+ 'such as a list or dictionary used as default value will be shared '
+ 'by\n'
+ 'all calls that don’t specify an argument value for the '
+ 'corresponding\n'
+ 'slot; this should usually be avoided.) If there are any unfilled\n'
+ 'slots for which no default value is specified, a "TypeError" '
+ 'exception\n'
+ 'is raised. Otherwise, the list of filled slots is used as the\n'
+ 'argument list for the call.\n'
'\n'
'**CPython implementation detail:** An implementation may provide\n'
'built-in functions whose positional parameters do not have names, '
@@ -1671,7 +1668,7 @@
'syntax\n'
'to be used in the same call, so in practice this confusion does '
'not\n'
- 'arise.\n'
+ 'often arise.\n'
'\n'
'If the syntax "**expression" appears in the function call,\n'
'"expression" must evaluate to a *mapping*, the contents of which '
@@ -2342,27 +2339,21 @@
'sequence\n'
'(such as a string, tuple or list) or other iterable object:\n'
'\n'
- ' for_stmt ::= "for" target_list "in" expression_list ":" '
- 'suite\n'
+ ' for_stmt ::= "for" target_list "in" starred_list ":" suite\n'
' ["else" ":" suite]\n'
'\n'
- 'The expression list is evaluated once; it should yield an '
- 'iterable\n'
- 'object. An iterator is created for the result of the\n'
- '"expression_list". The suite is then executed once for each '
+ 'The "starred_list" expression is evaluated once; it should yield '
+ 'an\n'
+ '*iterable* object. An *iterator* is created for that iterable. '
+ 'The\n'
+ 'first item provided by the iterator is then assigned to the '
+ 'target\n'
+ 'list using the standard rules for assignments (see Assignment\n'
+ 'statements), and the suite is executed. This repeats for each '
'item\n'
- 'provided by the iterator, in the order returned by the '
- 'iterator. Each\n'
- 'item in turn is assigned to the target list using the standard '
- 'rules\n'
- 'for assignments (see Assignment statements), and then the suite '
- 'is\n'
- 'executed. When the items are exhausted (which is immediately '
- 'when the\n'
- 'sequence is empty or an iterator raises a "StopIteration" '
- 'exception),\n'
- 'the suite in the "else" clause, if present, is executed, and the '
- 'loop\n'
+ 'provided by the iterator. When the iterator is exhausted, the '
+ 'suite\n'
+ 'in the "else" clause, if present, is executed, and the loop\n'
'terminates.\n'
'\n'
'A "break" statement executed in the first suite terminates the '
@@ -2397,6 +2388,10 @@
'iterating\n'
'"range(3)" successively yields 0, 1, and then 2.\n'
'\n'
+ 'Changed in version 3.11: Starred elements are now allowed in '
+ 'the\n'
+ 'expression list.\n'
+ '\n'
'\n'
'The "try" statement\n'
'===================\n'
@@ -2405,44 +2400,57 @@
'code\n'
'for a group of statements:\n'
'\n'
- ' try_stmt ::= try1_stmt | try2_stmt\n'
+ ' try_stmt ::= try1_stmt | try2_stmt | try3_stmt\n'
' try1_stmt ::= "try" ":" suite\n'
' ("except" [expression ["as" identifier]] ":" '
'suite)+\n'
' ["else" ":" suite]\n'
' ["finally" ":" suite]\n'
' try2_stmt ::= "try" ":" suite\n'
+ ' ("except" "*" expression ["as" identifier] ":" '
+ 'suite)+\n'
+ ' ["else" ":" suite]\n'
+ ' ["finally" ":" suite]\n'
+ ' try3_stmt ::= "try" ":" suite\n'
' "finally" ":" suite\n'
'\n'
+ 'Additional information on exceptions can be found in section\n'
+ 'Exceptions, and information on using the "raise" statement to '
+ 'generate\n'
+ 'exceptions may be found in section The raise statement.\n'
+ '\n'
+ '\n'
+ '"except" clause\n'
+ '---------------\n'
+ '\n'
'The "except" clause(s) specify one or more exception handlers. '
'When no\n'
'exception occurs in the "try" clause, no exception handler is\n'
'executed. When an exception occurs in the "try" suite, a search '
'for an\n'
- 'exception handler is started. This search inspects the except '
- 'clauses\n'
- 'in turn until one is found that matches the exception. An '
- 'expression-\n'
- 'less except clause, if present, must be last; it matches any\n'
- 'exception. For an except clause with an expression, that '
- 'expression\n'
- 'is evaluated, and the clause matches the exception if the '
- 'resulting\n'
- 'object is “compatible” with the exception. An object is '
- 'compatible\n'
- 'with an exception if the object is the class or a *non-virtual '
- 'base\n'
- 'class* of the exception object, or a tuple containing an item '
- 'that is\n'
- 'the class or a non-virtual base class of the exception object.\n'
- '\n'
- 'If no except clause matches the exception, the search for an '
+ 'exception handler is started. This search inspects the "except"\n'
+ 'clauses in turn until one is found that matches the exception. '
+ 'An\n'
+ 'expression-less "except" clause, if present, must be last; it '
+ 'matches\n'
+ 'any exception. For an "except" clause with an expression, that\n'
+ 'expression is evaluated, and the clause matches the exception if '
+ 'the\n'
+ 'resulting object is “compatible” with the exception. An object '
+ 'is\n'
+ 'compatible with an exception if the object is the class or a '
+ '*non-\n'
+ 'virtual base class* of the exception object, or a tuple '
+ 'containing an\n'
+ 'item that is the class or a non-virtual base class of the '
'exception\n'
- 'handler continues in the surrounding code and on the invocation '
- 'stack.\n'
- '[1]\n'
+ 'object.\n'
'\n'
- 'If the evaluation of an expression in the header of an except '
+ 'If no "except" clause matches the exception, the search for an\n'
+ 'exception handler continues in the surrounding code and on the\n'
+ 'invocation stack. [1]\n'
+ '\n'
+ 'If the evaluation of an expression in the header of an "except" '
'clause\n'
'raises an exception, the original search for a handler is '
'canceled and\n'
@@ -2452,24 +2460,24 @@
'raised\n'
'the exception).\n'
'\n'
- 'When a matching except clause is found, the exception is '
+ 'When a matching "except" clause is found, the exception is '
'assigned to\n'
- 'the target specified after the "as" keyword in that except '
- 'clause, if\n'
- 'present, and the except clause’s suite is executed. All except\n'
- 'clauses must have an executable block. When the end of this '
+ 'the target specified after the "as" keyword in that "except" '
+ 'clause,\n'
+ 'if present, and the "except" clause’s suite is executed. All '
+ '"except"\n'
+ 'clauses must have an executable block. When the end of this '
'block is\n'
- 'reached, execution continues normally after the entire try '
- 'statement.\n'
- '(This means that if two nested handlers exist for the same '
- 'exception,\n'
- 'and the exception occurs in the try clause of the inner handler, '
- 'the\n'
- 'outer handler will not handle the exception.)\n'
+ 'reached, execution continues normally after the entire "try"\n'
+ 'statement. (This means that if two nested handlers exist for the '
+ 'same\n'
+ 'exception, and the exception occurs in the "try" clause of the '
+ 'inner\n'
+ 'handler, the outer handler will not handle the exception.)\n'
'\n'
'When an exception has been assigned using "as target", it is '
'cleared\n'
- 'at the end of the except clause. This is as if\n'
+ 'at the end of the "except" clause. This is as if\n'
'\n'
' except E as N:\n'
' foo\n'
@@ -2484,7 +2492,7 @@
'\n'
'This means the exception must be assigned to a different name to '
'be\n'
- 'able to refer to it after the except clause. Exceptions are '
+ 'able to refer to it after the "except" clause. Exceptions are '
'cleared\n'
'because with the traceback attached to them, they form a '
'reference\n'
@@ -2492,41 +2500,113 @@
'alive\n'
'until the next garbage collection occurs.\n'
'\n'
- 'Before an except clause’s suite is executed, details about the\n'
- 'exception are stored in the "sys" module and can be accessed '
- 'via\n'
- '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting '
- 'of the\n'
- 'exception class, the exception instance and a traceback object '
- '(see\n'
- 'section The standard type hierarchy) identifying the point in '
- 'the\n'
- 'program where the exception occurred. The details about the '
- 'exception\n'
- 'accessed via "sys.exc_info()" are restored to their previous '
- 'values\n'
- 'when leaving an exception handler:\n'
+ 'Before an "except" clause’s suite is executed, the exception is '
+ 'stored\n'
+ 'in the "sys" module, where it can be accessed from within the '
+ 'body of\n'
+ 'the "except" clause by calling "sys.exception()". When leaving '
+ 'an\n'
+ 'exception handler, the exception stored in the "sys" module is '
+ 'reset\n'
+ 'to its previous value:\n'
'\n'
- ' >>> print(sys.exc_info())\n'
- ' (None, None, None)\n'
+ ' >>> print(sys.exception())\n'
+ ' None\n'
' >>> try:\n'
' ... raise TypeError\n'
' ... except:\n'
- ' ... print(sys.exc_info())\n'
+ ' ... print(repr(sys.exception()))\n'
' ... try:\n'
' ... raise ValueError\n'
' ... except:\n'
- ' ... print(sys.exc_info())\n'
- ' ... print(sys.exc_info())\n'
+ ' ... print(repr(sys.exception()))\n'
+ ' ... print(repr(sys.exception()))\n'
' ...\n'
- " (<class 'TypeError'>, TypeError(), <traceback object at "
- '0x10efad080>)\n'
- " (<class 'ValueError'>, ValueError(), <traceback object at "
- '0x10efad040>)\n'
- " (<class 'TypeError'>, TypeError(), <traceback object at "
- '0x10efad080>)\n'
- ' >>> print(sys.exc_info())\n'
- ' (None, None, None)\n'
+ ' TypeError()\n'
+ ' ValueError()\n'
+ ' TypeError()\n'
+ ' >>> print(sys.exception())\n'
+ ' None\n'
+ '\n'
+ '\n'
+ '"except*" clause\n'
+ '----------------\n'
+ '\n'
+ 'The "except*" clause(s) are used for handling "ExceptionGroup"s. '
+ 'The\n'
+ 'exception type for matching is interpreted as in the case of '
+ '"except",\n'
+ 'but in the case of exception groups we can have partial matches '
+ 'when\n'
+ 'the type matches some of the exceptions in the group. This means '
+ 'that\n'
+ 'multiple "except*" clauses can execute, each handling part of '
+ 'the\n'
+ 'exception group. Each clause executes at most once and handles '
+ 'an\n'
+ 'exception group of all matching exceptions. Each exception in '
+ 'the\n'
+ 'group is handled by at most one "except*" clause, the first '
+ 'that\n'
+ 'matches it.\n'
+ '\n'
+ ' >>> try:\n'
+ ' ... raise ExceptionGroup("eg",\n'
+ ' ... [ValueError(1), TypeError(2), OSError(3), '
+ 'OSError(4)])\n'
+ ' ... except* TypeError as e:\n'
+ " ... print(f'caught {type(e)} with nested "
+ "{e.exceptions}')\n"
+ ' ... except* OSError as e:\n'
+ " ... print(f'caught {type(e)} with nested "
+ "{e.exceptions}')\n"
+ ' ...\n'
+ " caught <class 'ExceptionGroup'> with nested (TypeError(2),)\n"
+ " caught <class 'ExceptionGroup'> with nested (OSError(3), "
+ 'OSError(4))\n'
+ ' + Exception Group Traceback (most recent call last):\n'
+ ' | File "<stdin>", line 2, in <module>\n'
+ ' | ExceptionGroup: eg\n'
+ ' +-+---------------- 1 ----------------\n'
+ ' | ValueError: 1\n'
+ ' +------------------------------------\n'
+ '\n'
+ 'Any remaining exceptions that were not handled by any "except*" '
+ 'clause\n'
+ 'are re-raised at the end, combined into an exception group along '
+ 'with\n'
+ 'all exceptions that were raised from within "except*" clauses.\n'
+ '\n'
+ 'From version 3.11.4, when the entire "ExceptionGroup" is handled '
+ 'and\n'
+ 'only one exception is raised from an "except*" clause, this '
+ 'exception\n'
+ 'is no longer wrapped to form a new "ExceptionGroup".\n'
+ '\n'
+ 'If the raised exception is not an exception group and its type '
+ 'matches\n'
+ 'one of the "except*" clauses, it is caught and wrapped by an '
+ 'exception\n'
+ 'group with an empty message string.\n'
+ '\n'
+ ' >>> try:\n'
+ ' ... raise BlockingIOError\n'
+ ' ... except* BlockingIOError as e:\n'
+ ' ... print(repr(e))\n'
+ ' ...\n'
+ " ExceptionGroup('', (BlockingIOError()))\n"
+ '\n'
+ 'An "except*" clause must have a matching type, and this type '
+ 'cannot be\n'
+ 'a subclass of "BaseExceptionGroup". It is not possible to mix '
+ '"except"\n'
+ 'and "except*" in the same "try". "break", "continue" and '
+ '"return"\n'
+ 'cannot appear in an "except*" clause.\n'
+ '\n'
+ '\n'
+ '"else" clause\n'
+ '-------------\n'
'\n'
'The optional "else" clause is executed if the control flow '
'leaves the\n'
@@ -2536,6 +2616,10 @@
'are\n'
'not handled by the preceding "except" clauses.\n'
'\n'
+ '\n'
+ '"finally" clause\n'
+ '----------------\n'
+ '\n'
'If "finally" is present, it specifies a ‘cleanup’ handler. The '
'"try"\n'
'clause is executed, including any "except" and "else" clauses. '
@@ -2589,11 +2673,6 @@
' >>> foo()\n'
" 'finally'\n"
'\n'
- 'Additional information on exceptions can be found in section\n'
- 'Exceptions, and information on using the "raise" statement to '
- 'generate\n'
- 'exceptions may be found in section The raise statement.\n'
- '\n'
'Changed in version 3.8: Prior to Python 3.8, a "continue" '
'statement\n'
'was illegal in the "finally" clause due to a problem with the\n'
@@ -4514,8 +4593,7 @@
'case\n'
' performance of a dict insertion, O(n^2) complexity. '
'See\n'
- ' http://www.ocert.org/advisories/ocert-2011-003.html '
- 'for\n'
+ ' http://ocert.org/advisories/ocert-2011-003.html for\n'
' details.Changing hash values affects the iteration '
'order of sets.\n'
' Python has never made guarantees about this ordering '
@@ -4578,60 +4656,14 @@
'traces of\n'
' Python programs.\n'
'\n'
- 'The debugger’s prompt is "(Pdb)". Typical usage to run a program '
- 'under\n'
- 'control of the debugger is:\n'
- '\n'
- ' >>> import pdb\n'
- ' >>> import mymodule\n'
- " >>> pdb.run('mymodule.test()')\n"
- ' > <string>(0)?()\n'
- ' (Pdb) continue\n'
- ' > <string>(1)?()\n'
- ' (Pdb) continue\n'
- " NameError: 'spam'\n"
- ' > <string>(1)?()\n'
- ' (Pdb)\n'
- '\n'
- 'Changed in version 3.3: Tab-completion via the "readline" module '
- 'is\n'
- 'available for commands and command arguments, e.g. the current '
- 'global\n'
- 'and local names are offered as arguments of the "p" command.\n'
- '\n'
- '"pdb.py" can also be invoked as a script to debug other '
- 'scripts. For\n'
- 'example:\n'
- '\n'
- ' python -m pdb myscript.py\n'
- '\n'
- 'When invoked as a script, pdb will automatically enter '
- 'post-mortem\n'
- 'debugging if the program being debugged exits abnormally. After '
- 'post-\n'
- 'mortem debugging (or after normal exit of the program), pdb '
- 'will\n'
- 'restart the program. Automatic restarting preserves pdb’s state '
- '(such\n'
- 'as breakpoints) and in most cases is more useful than quitting '
- 'the\n'
- 'debugger upon program’s exit.\n'
- '\n'
- 'New in version 3.2: "pdb.py" now accepts a "-c" option that '
- 'executes\n'
- 'commands as if given in a ".pdbrc" file, see Debugger Commands.\n'
- '\n'
- 'New in version 3.7: "pdb.py" now accepts a "-m" option that '
- 'execute\n'
- 'modules similar to the way "python -m" does. As with a script, '
- 'the\n'
- 'debugger will pause execution just before the first line of the\n'
- 'module.\n'
- '\n'
'The typical usage to break into the debugger is to insert:\n'
'\n'
' import pdb; pdb.set_trace()\n'
'\n'
+ 'Or:\n'
+ '\n'
+ ' breakpoint()\n'
+ '\n'
'at the location you want to break into the debugger, and then '
'run the\n'
'program. You can then step through the code following this '
@@ -4643,21 +4675,83 @@
'with\n'
'defaults, can be used instead of "import pdb; pdb.set_trace()".\n'
'\n'
+ ' def double(x):\n'
+ ' breakpoint()\n'
+ ' return x * 2\n'
+ ' val = 3\n'
+ ' print(f"{val} * 2 is {double(val)}")\n'
+ '\n'
+ 'The debugger’s prompt is "(Pdb)", which is the indicator that '
+ 'you are\n'
+ 'in debug mode:\n'
+ '\n'
+ ' > ...(3)double()\n'
+ ' -> return x * 2\n'
+ ' (Pdb) p x\n'
+ ' 3\n'
+ ' (Pdb) continue\n'
+ ' 3 * 2 is 6\n'
+ '\n'
+ 'Changed in version 3.3: Tab-completion via the "readline" module '
+ 'is\n'
+ 'available for commands and command arguments, e.g. the current '
+ 'global\n'
+ 'and local names are offered as arguments of the "p" command.\n'
+ '\n'
+ 'You can also invoke "pdb" from the command line to debug other\n'
+ 'scripts. For example:\n'
+ '\n'
+ ' python -m pdb myscript.py\n'
+ '\n'
+ 'When invoked as a module, pdb will automatically enter '
+ 'post-mortem\n'
+ 'debugging if the program being debugged exits abnormally. After '
+ 'post-\n'
+ 'mortem debugging (or after normal exit of the program), pdb '
+ 'will\n'
+ 'restart the program. Automatic restarting preserves pdb’s state '
+ '(such\n'
+ 'as breakpoints) and in most cases is more useful than quitting '
+ 'the\n'
+ 'debugger upon program’s exit.\n'
+ '\n'
+ 'New in version 3.2: "-c" option is introduced to execute '
+ 'commands as\n'
+ 'if given in a ".pdbrc" file, see Debugger Commands.\n'
+ '\n'
+ 'New in version 3.7: "-m" option is introduced to execute '
+ 'modules\n'
+ 'similar to the way "python -m" does. As with a script, the '
+ 'debugger\n'
+ 'will pause execution just before the first line of the module.\n'
+ '\n'
+ 'Typical usage to execute a statement under control of the '
+ 'debugger is:\n'
+ '\n'
+ ' >>> import pdb\n'
+ ' >>> def f(x):\n'
+ ' ... print(1 / x)\n'
+ ' >>> pdb.run("f(2)")\n'
+ ' > <string>(1)<module>()\n'
+ ' (Pdb) continue\n'
+ ' 0.5\n'
+ ' >>>\n'
+ '\n'
'The typical usage to inspect a crashed program is:\n'
'\n'
' >>> import pdb\n'
- ' >>> import mymodule\n'
- ' >>> mymodule.test()\n'
+ ' >>> def f(x):\n'
+ ' ... print(1 / x)\n'
+ ' ...\n'
+ ' >>> f(0)\n'
' Traceback (most recent call last):\n'
' File "<stdin>", line 1, in <module>\n'
- ' File "./mymodule.py", line 4, in test\n'
- ' test2()\n'
- ' File "./mymodule.py", line 3, in test2\n'
- ' print(spam)\n'
- ' NameError: spam\n'
+ ' File "<stdin>", line 2, in f\n'
+ ' ZeroDivisionError: division by zero\n'
' >>> pdb.pm()\n'
- ' > ./mymodule.py(3)test2()\n'
- ' -> print(spam)\n'
+ ' > <stdin>(2)f()\n'
+ ' (Pdb) p x\n'
+ ' 0\n'
' (Pdb)\n'
'\n'
'The module defines the following functions; each enters the '
@@ -4843,13 +4937,18 @@
'\n'
'If a file ".pdbrc" exists in the user’s home directory or in '
'the\n'
- 'current directory, it is read in and executed as if it had been '
- 'typed\n'
- 'at the debugger prompt. This is particularly useful for '
- 'aliases. If\n'
- 'both files exist, the one in the home directory is read first '
- 'and\n'
- 'aliases defined there can be overridden by the local file.\n'
+ 'current directory, it is read with "\'utf-8\'" encoding and '
+ 'executed as\n'
+ 'if it had been typed at the debugger prompt. This is '
+ 'particularly\n'
+ 'useful for aliases. If both files exist, the one in the home\n'
+ 'directory is read first and aliases defined there can be '
+ 'overridden by\n'
+ 'the local file.\n'
+ '\n'
+ 'Changed in version 3.11: ".pdbrc" is now read with "\'utf-8\'" '
+ 'encoding.\n'
+ 'Previously, it was read with the system locale encoding.\n'
'\n'
'Changed in version 3.2: ".pdbrc" can now contain commands that\n'
'continue debugging, such as "continue" or "next". Previously, '
@@ -4871,9 +4970,9 @@
'\n'
' Print a stack trace, with the most recent frame at the '
'bottom. An\n'
- ' arrow indicates the current frame, which determines the '
- 'context of\n'
- ' most commands.\n'
+ ' arrow (">") indicates the current frame, which determines '
+ 'the\n'
+ ' context of most commands.\n'
'\n'
'd(own) [count]\n'
'\n'
@@ -5101,7 +5200,9 @@
'\n'
'a(rgs)\n'
'\n'
- ' Print the argument list of the current function.\n'
+ ' Print the arguments of the current function and their '
+ 'current\n'
+ ' values.\n'
'\n'
'p expression\n'
'\n'
@@ -5139,6 +5240,54 @@
'current\n'
' frame.\n'
'\n'
+ ' Note:\n'
+ '\n'
+ ' Display evaluates *expression* and compares to the result '
+ 'of the\n'
+ ' previous evaluation of *expression*, so when the result is\n'
+ ' mutable, display may not be able to pick up the changes.\n'
+ '\n'
+ ' Example:\n'
+ '\n'
+ ' lst = []\n'
+ ' breakpoint()\n'
+ ' pass\n'
+ ' lst.append(1)\n'
+ ' print(lst)\n'
+ '\n'
+ ' Display won’t realize "lst" has been changed because the '
+ 'result of\n'
+ ' evaluation is modified in place by "lst.append(1)" before '
+ 'being\n'
+ ' compared:\n'
+ '\n'
+ ' > example.py(3)<module>()\n'
+ ' -> pass\n'
+ ' (Pdb) display lst\n'
+ ' display lst: []\n'
+ ' (Pdb) n\n'
+ ' > example.py(4)<module>()\n'
+ ' -> lst.append(1)\n'
+ ' (Pdb) n\n'
+ ' > example.py(5)<module>()\n'
+ ' -> print(lst)\n'
+ ' (Pdb)\n'
+ '\n'
+ ' You can do some tricks with copy mechanism to make it work:\n'
+ '\n'
+ ' > example.py(3)<module>()\n'
+ ' -> pass\n'
+ ' (Pdb) display lst[:]\n'
+ ' display lst[:]: []\n'
+ ' (Pdb) n\n'
+ ' > example.py(4)<module>()\n'
+ ' -> lst.append(1)\n'
+ ' (Pdb) n\n'
+ ' > example.py(5)<module>()\n'
+ ' -> print(lst)\n'
+ ' display lst[:]: [1] [old: []]\n'
+ ' (Pdb)\n'
+ '\n'
' New in version 3.2.\n'
'\n'
'undisplay [expression]\n'
@@ -5240,7 +5389,8 @@
'\n'
'retval\n'
'\n'
- ' Print the return value for the last return of a function.\n'
+ ' Print the return value for the last return of the current '
+ 'function.\n'
'\n'
'-[ Footnotes ]-\n'
'\n'
@@ -5510,9 +5660,10 @@
'\n'
' * "for" loop header,\n'
'\n'
- ' * after "as" in a "with" statement, "except" clause or in the '
- 'as-\n'
- ' pattern in structural pattern matching,\n'
+ ' * after "as" in a "with" statement, "except" clause, '
+ '"except*"\n'
+ ' clause, or in the as-pattern in structural pattern '
+ 'matching,\n'
'\n'
' * in a capture pattern in structural pattern matching\n'
'\n'
@@ -5861,22 +6012,16 @@
'sequence\n'
'(such as a string, tuple or list) or other iterable object:\n'
'\n'
- ' for_stmt ::= "for" target_list "in" expression_list ":" suite\n'
+ ' for_stmt ::= "for" target_list "in" starred_list ":" suite\n'
' ["else" ":" suite]\n'
'\n'
- 'The expression list is evaluated once; it should yield an iterable\n'
- 'object. An iterator is created for the result of the\n'
- '"expression_list". The suite is then executed once for each item\n'
- 'provided by the iterator, in the order returned by the iterator. '
- 'Each\n'
- 'item in turn is assigned to the target list using the standard rules\n'
- 'for assignments (see Assignment statements), and then the suite is\n'
- 'executed. When the items are exhausted (which is immediately when '
- 'the\n'
- 'sequence is empty or an iterator raises a "StopIteration" '
- 'exception),\n'
- 'the suite in the "else" clause, if present, is executed, and the '
- 'loop\n'
+ 'The "starred_list" expression is evaluated once; it should yield an\n'
+ '*iterable* object. An *iterator* is created for that iterable. The\n'
+ 'first item provided by the iterator is then assigned to the target\n'
+ 'list using the standard rules for assignments (see Assignment\n'
+ 'statements), and the suite is executed. This repeats for each item\n'
+ 'provided by the iterator. When the iterator is exhausted, the suite\n'
+ 'in the "else" clause, if present, is executed, and the loop\n'
'terminates.\n'
'\n'
'A "break" statement executed in the first suite terminates the loop\n'
@@ -5902,7 +6047,10 @@
'but if the sequence is empty, they will not have been assigned to at\n'
'all by the loop. Hint: the built-in type "range()" represents\n'
'immutable arithmetic sequences of integers. For instance, iterating\n'
- '"range(3)" successively yields 0, 1, and then 2.\n',
+ '"range(3)" successively yields 0, 1, and then 2.\n'
+ '\n'
+ 'Changed in version 3.11: Starred elements are now allowed in the\n'
+ 'expression list.\n',
'formatstrings': 'Format String Syntax\n'
'********************\n'
'\n'
@@ -6095,7 +6243,8 @@
'The general form of a *standard format specifier* is:\n'
'\n'
' format_spec ::= '
- '[[fill]align][sign][#][0][width][grouping_option][.precision][type]\n'
+ '[[fill]align][sign]["z"]["#"]["0"][width][grouping_option]["." '
+ 'precision][type]\n'
' fill ::= <any character>\n'
' align ::= "<" | ">" | "=" | "^"\n'
' sign ::= "+" | "-" | " "\n'
@@ -6197,6 +6346,16 @@
' '
'+-----------+------------------------------------------------------------+\n'
'\n'
+ 'The "\'z\'" option coerces negative zero floating-point '
+ 'values to\n'
+ 'positive zero after rounding to the format precision. This '
+ 'option is\n'
+ 'only valid for floating-point presentation types.\n'
+ '\n'
+ 'Changed in version 3.11: Added the "\'z\'" option (see also '
+ '**PEP\n'
+ '682**).\n'
+ '\n'
'The "\'#\'" option causes the “alternate form” to be used '
'for the\n'
'conversion. The alternate form is defined differently for '
@@ -7066,8 +7225,8 @@
'\n'
'A non-normative HTML file listing all valid identifier '
'characters for\n'
- 'Unicode 4.1 can be found at\n'
- 'https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt\n'
+ 'Unicode 14.0.0 can be found at\n'
+ 'https://www.unicode.org/Public/14.0.0/ucd/DerivedCoreProperties.txt\n'
'\n'
'\n'
'Keywords\n'
@@ -7620,9 +7779,8 @@
'\n'
' * "for" loop header,\n'
'\n'
- ' * after "as" in a "with" statement, "except" clause or in the '
- 'as-\n'
- ' pattern in structural pattern matching,\n'
+ ' * after "as" in a "with" statement, "except" clause, "except*"\n'
+ ' clause, or in the as-pattern in structural pattern matching,\n'
'\n'
' * in a capture pattern in structural pattern matching\n'
'\n'
@@ -7890,11 +8048,11 @@
'instance, to\n'
' evaluate the expression "x + y", where *x* is an '
'instance of a\n'
- ' class that has an "__add__()" method, "x.__add__(y)" is '
- 'called.\n'
- ' The "__divmod__()" method should be the equivalent to '
- 'using\n'
- ' "__floordiv__()" and "__mod__()"; it should not be '
+ ' class that has an "__add__()" method, '
+ '"type(x).__add__(x, y)" is\n'
+ ' called. The "__divmod__()" method should be the '
+ 'equivalent to\n'
+ ' using "__floordiv__()" and "__mod__()"; it should not be '
'related to\n'
' "__truediv__()". Note that "__pow__()" should be '
'defined to accept\n'
@@ -7935,9 +8093,9 @@
'expression "x -\n'
' y", where *y* is an instance of a class that has an '
'"__rsub__()"\n'
- ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" '
- 'returns\n'
- ' *NotImplemented*.\n'
+ ' method, "type(y).__rsub__(y, x)" is called if '
+ '"type(x).__sub__(x,\n'
+ ' y)" returns *NotImplemented*.\n'
'\n'
' Note that ternary "pow()" will not try calling '
'"__rpow__()" (the\n'
@@ -8050,7 +8208,11 @@
'\n'
' The built-in function "int()" falls back to '
'"__trunc__()" if\n'
- ' neither "__int__()" nor "__index__()" is defined.\n',
+ ' neither "__int__()" nor "__index__()" is defined.\n'
+ '\n'
+ ' Changed in version 3.11: The delegation of "int()" to '
+ '"__trunc__()"\n'
+ ' is deprecated.\n',
'objects': 'Objects, values and types\n'
'*************************\n'
'\n'
@@ -8191,8 +8353,9 @@
'the syntax is explicitly given, operators are binary. '
'Operators in\n'
'the same box group left to right (except for '
- 'exponentiation, which\n'
- 'groups from right to left).\n'
+ 'exponentiation and\n'
+ 'conditional expressions, which group from right to '
+ 'left).\n'
'\n'
'Note that comparisons, membership tests, and identity '
'tests, all have\n'
@@ -8521,7 +8684,14 @@
'\n'
'New in version 3.3: The "__suppress_context__" attribute to '
'suppress\n'
- 'automatic display of the exception context.\n',
+ 'automatic display of the exception context.\n'
+ '\n'
+ 'Changed in version 3.11: If the traceback of the active exception '
+ 'is\n'
+ 'modified in an "except" clause, a subsequent "raise" statement re-\n'
+ 'raises the exception with the modified traceback. Previously, the\n'
+ 'exception was re-raised with the traceback it had when it was '
+ 'caught.\n',
'return': 'The "return" statement\n'
'**********************\n'
'\n'
@@ -9408,8 +9578,7 @@
' by carefully chosen inputs that exploit the worst case\n'
' performance of a dict insertion, O(n^2) complexity. '
'See\n'
- ' http://www.ocert.org/advisories/ocert-2011-003.html '
- 'for\n'
+ ' http://ocert.org/advisories/ocert-2011-003.html for\n'
' details.Changing hash values affects the iteration '
'order of sets.\n'
' Python has never made guarantees about this ordering '
@@ -9752,13 +9921,12 @@
' "A.__dict__[\'x\'].__get__(None, A)".\n'
'\n'
'Super Binding\n'
- ' If "a" is an instance of "super", then the binding '
- '"super(B,\n'
- ' obj).m()" searches "obj.__class__.__mro__" for the base '
- 'class "A"\n'
- ' immediately following "B" and then invokes the descriptor '
- 'with the\n'
- ' call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n'
+ ' A dotted lookup such as "super(A, a).x" searches\n'
+ ' "a.__class__.__mro__" for a base class "B" following "A" '
+ 'and then\n'
+ ' returns "B.__dict__[\'x\'].__get__(a, A)". If not a '
+ 'descriptor, "x"\n'
+ ' is returned unchanged.\n'
'\n'
'For instance bindings, the precedence of descriptor '
'invocation depends\n'
@@ -10064,20 +10232,32 @@
'Resolving MRO entries\n'
'---------------------\n'
'\n'
- 'If a base that appears in class definition is not an '
+ 'object.__mro_entries__(self, bases)\n'
+ '\n'
+ ' If a base that appears in a class definition is not an '
'instance of\n'
- '"type", then an "__mro_entries__" method is searched on it. '
- 'If found,\n'
- 'it is called with the original bases tuple. This method must '
- 'return a\n'
- 'tuple of classes that will be used instead of this base. The '
- 'tuple may\n'
- 'be empty, in such case the original base is ignored.\n'
+ ' "type", then an "__mro_entries__()" method is searched on '
+ 'the base.\n'
+ ' If an "__mro_entries__()" method is found, the base is '
+ 'substituted\n'
+ ' with the result of a call to "__mro_entries__()" when '
+ 'creating the\n'
+ ' class. The method is called with the original bases tuple '
+ 'passed to\n'
+ ' the *bases* parameter, and must return a tuple of classes '
+ 'that will\n'
+ ' be used instead of the base. The returned tuple may be '
+ 'empty: in\n'
+ ' these cases, the original base is ignored.\n'
'\n'
'See also:\n'
'\n'
- ' **PEP 560** - Core support for typing module and generic '
- 'types\n'
+ ' "types.resolve_bases()"\n'
+ ' Dynamically resolve bases that are not instances of '
+ '"type".\n'
+ '\n'
+ ' **PEP 560**\n'
+ ' Core support for typing module and generic types.\n'
'\n'
'\n'
'Determining the appropriate metaclass\n'
@@ -10407,7 +10587,7 @@
' from inspect import isclass\n'
'\n'
' def subscribe(obj, x):\n'
- ' """Return the result of the expression `obj[x]`"""\n'
+ ' """Return the result of the expression \'obj[x]\'"""\n'
'\n'
' class_of_obj = type(obj)\n'
'\n'
@@ -10777,11 +10957,11 @@
'to\n'
' evaluate the expression "x + y", where *x* is an instance '
'of a\n'
- ' class that has an "__add__()" method, "x.__add__(y)" is '
- 'called.\n'
- ' The "__divmod__()" method should be the equivalent to '
- 'using\n'
- ' "__floordiv__()" and "__mod__()"; it should not be '
+ ' class that has an "__add__()" method, "type(x).__add__(x, '
+ 'y)" is\n'
+ ' called. The "__divmod__()" method should be the '
+ 'equivalent to\n'
+ ' using "__floordiv__()" and "__mod__()"; it should not be '
'related to\n'
' "__truediv__()". Note that "__pow__()" should be defined '
'to accept\n'
@@ -10822,9 +11002,9 @@
'expression "x -\n'
' y", where *y* is an instance of a class that has an '
'"__rsub__()"\n'
- ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" '
- 'returns\n'
- ' *NotImplemented*.\n'
+ ' method, "type(y).__rsub__(y, x)" is called if '
+ '"type(x).__sub__(x,\n'
+ ' y)" returns *NotImplemented*.\n'
'\n'
' Note that ternary "pow()" will not try calling '
'"__rpow__()" (the\n'
@@ -10939,6 +11119,10 @@
'if\n'
' neither "__int__()" nor "__index__()" is defined.\n'
'\n'
+ ' Changed in version 3.11: The delegation of "int()" to '
+ '"__trunc__()"\n'
+ ' is deprecated.\n'
+ '\n'
'\n'
'With Statement Context Managers\n'
'===============================\n'
@@ -11729,7 +11913,7 @@
'followed by\n'
' the string itself.\n'
'\n'
- 'str.rsplit(sep=None, maxsplit=-1)\n'
+ 'str.rsplit(sep=None, maxsplit=- 1)\n'
'\n'
' Return a list of the words in the string, using *sep* '
'as the\n'
@@ -11770,7 +11954,7 @@
" >>> 'Monty Python'.removesuffix(' Python')\n"
" 'Monty'\n"
'\n'
- 'str.split(sep=None, maxsplit=-1)\n'
+ 'str.split(sep=None, maxsplit=- 1)\n'
'\n'
' Return a list of the words in the string, using *sep* '
'as the\n'
@@ -12125,15 +12309,12 @@
'single quotes ("\'") or double quotes ("""). They can also be '
'enclosed\n'
'in matching groups of three single or double quotes (these are\n'
- 'generally referred to as *triple-quoted strings*). The backslash '
- '("\\")\n'
- 'character is used to give special meaning to otherwise ordinary\n'
- 'characters like "n", which means ‘newline’ when escaped ("\\n"). '
- 'It can\n'
- 'also be used to escape characters that otherwise have a special\n'
- 'meaning, such as newline, backslash itself, or the quote '
+ 'generally referred to as *triple-quoted strings*). The '
+ 'backslash\n'
+ '("\\") character is used to escape characters that otherwise have '
+ 'a\n'
+ 'special meaning, such as newline, backslash itself, or the quote\n'
'character.\n'
- 'See escape sequences below for examples.\n'
'\n'
'Bytes literals are always prefixed with "\'b\'" or "\'B\'"; they '
'produce\n'
@@ -12268,6 +12449,12 @@
'\n'
'2. As in Standard C, up to three octal digits are accepted.\n'
'\n'
+ ' Changed in version 3.11: Octal escapes with value larger than\n'
+ ' "0o377" produce a "DeprecationWarning". In a future Python '
+ 'version\n'
+ ' they will be a "SyntaxWarning" and eventually a '
+ '"SyntaxError".\n'
+ '\n'
'3. Unlike in Standard C, exactly two hex digits are required.\n'
'\n'
'4. In a bytes literal, hexadecimal and octal escapes denote the '
@@ -12438,40 +12625,53 @@
'The "try" statement specifies exception handlers and/or cleanup code\n'
'for a group of statements:\n'
'\n'
- ' try_stmt ::= try1_stmt | try2_stmt\n'
+ ' try_stmt ::= try1_stmt | try2_stmt | try3_stmt\n'
' try1_stmt ::= "try" ":" suite\n'
' ("except" [expression ["as" identifier]] ":" '
'suite)+\n'
' ["else" ":" suite]\n'
' ["finally" ":" suite]\n'
' try2_stmt ::= "try" ":" suite\n'
+ ' ("except" "*" expression ["as" identifier] ":" '
+ 'suite)+\n'
+ ' ["else" ":" suite]\n'
+ ' ["finally" ":" suite]\n'
+ ' try3_stmt ::= "try" ":" suite\n'
' "finally" ":" suite\n'
'\n'
+ 'Additional information on exceptions can be found in section\n'
+ 'Exceptions, and information on using the "raise" statement to '
+ 'generate\n'
+ 'exceptions may be found in section The raise statement.\n'
+ '\n'
+ '\n'
+ '"except" clause\n'
+ '===============\n'
+ '\n'
'The "except" clause(s) specify one or more exception handlers. When '
'no\n'
'exception occurs in the "try" clause, no exception handler is\n'
'executed. When an exception occurs in the "try" suite, a search for '
'an\n'
- 'exception handler is started. This search inspects the except '
- 'clauses\n'
- 'in turn until one is found that matches the exception. An '
- 'expression-\n'
- 'less except clause, if present, must be last; it matches any\n'
- 'exception. For an except clause with an expression, that expression\n'
- 'is evaluated, and the clause matches the exception if the resulting\n'
- 'object is “compatible” with the exception. An object is compatible\n'
- 'with an exception if the object is the class or a *non-virtual base\n'
- 'class* of the exception object, or a tuple containing an item that '
- 'is\n'
- 'the class or a non-virtual base class of the exception object.\n'
+ 'exception handler is started. This search inspects the "except"\n'
+ 'clauses in turn until one is found that matches the exception. An\n'
+ 'expression-less "except" clause, if present, must be last; it '
+ 'matches\n'
+ 'any exception. For an "except" clause with an expression, that\n'
+ 'expression is evaluated, and the clause matches the exception if the\n'
+ 'resulting object is “compatible” with the exception. An object is\n'
+ 'compatible with an exception if the object is the class or a *non-\n'
+ 'virtual base class* of the exception object, or a tuple containing '
+ 'an\n'
+ 'item that is the class or a non-virtual base class of the exception\n'
+ 'object.\n'
'\n'
- 'If no except clause matches the exception, the search for an '
- 'exception\n'
- 'handler continues in the surrounding code and on the invocation '
- 'stack.\n'
- '[1]\n'
+ 'If no "except" clause matches the exception, the search for an\n'
+ 'exception handler continues in the surrounding code and on the\n'
+ 'invocation stack. [1]\n'
'\n'
- 'If the evaluation of an expression in the header of an except clause\n'
+ 'If the evaluation of an expression in the header of an "except" '
+ 'clause\n'
'raises an exception, the original search for a handler is canceled '
'and\n'
'a search starts for the new exception in the surrounding code and on\n'
@@ -12479,21 +12679,20 @@
'raised\n'
'the exception).\n'
'\n'
- 'When a matching except clause is found, the exception is assigned to\n'
- 'the target specified after the "as" keyword in that except clause, '
- 'if\n'
- 'present, and the except clause’s suite is executed. All except\n'
- 'clauses must have an executable block. When the end of this block '
- 'is\n'
- 'reached, execution continues normally after the entire try '
- 'statement.\n'
- '(This means that if two nested handlers exist for the same '
- 'exception,\n'
- 'and the exception occurs in the try clause of the inner handler, the\n'
- 'outer handler will not handle the exception.)\n'
+ 'When a matching "except" clause is found, the exception is assigned '
+ 'to\n'
+ 'the target specified after the "as" keyword in that "except" clause,\n'
+ 'if present, and the "except" clause’s suite is executed. All '
+ '"except"\n'
+ 'clauses must have an executable block. When the end of this block is\n'
+ 'reached, execution continues normally after the entire "try"\n'
+ 'statement. (This means that if two nested handlers exist for the '
+ 'same\n'
+ 'exception, and the exception occurs in the "try" clause of the inner\n'
+ 'handler, the outer handler will not handle the exception.)\n'
'\n'
'When an exception has been assigned using "as target", it is cleared\n'
- 'at the end of the except clause. This is as if\n'
+ 'at the end of the "except" clause. This is as if\n'
'\n'
' except E as N:\n'
' foo\n'
@@ -12507,42 +12706,107 @@
' del N\n'
'\n'
'This means the exception must be assigned to a different name to be\n'
- 'able to refer to it after the except clause. Exceptions are cleared\n'
+ 'able to refer to it after the "except" clause. Exceptions are '
+ 'cleared\n'
'because with the traceback attached to them, they form a reference\n'
'cycle with the stack frame, keeping all locals in that frame alive\n'
'until the next garbage collection occurs.\n'
'\n'
- 'Before an except clause’s suite is executed, details about the\n'
- 'exception are stored in the "sys" module and can be accessed via\n'
- '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of '
- 'the\n'
- 'exception class, the exception instance and a traceback object (see\n'
- 'section The standard type hierarchy) identifying the point in the\n'
- 'program where the exception occurred. The details about the '
- 'exception\n'
- 'accessed via "sys.exc_info()" are restored to their previous values\n'
- 'when leaving an exception handler:\n'
+ 'Before an "except" clause’s suite is executed, the exception is '
+ 'stored\n'
+ 'in the "sys" module, where it can be accessed from within the body '
+ 'of\n'
+ 'the "except" clause by calling "sys.exception()". When leaving an\n'
+ 'exception handler, the exception stored in the "sys" module is reset\n'
+ 'to its previous value:\n'
'\n'
- ' >>> print(sys.exc_info())\n'
- ' (None, None, None)\n'
+ ' >>> print(sys.exception())\n'
+ ' None\n'
' >>> try:\n'
' ... raise TypeError\n'
' ... except:\n'
- ' ... print(sys.exc_info())\n'
+ ' ... print(repr(sys.exception()))\n'
' ... try:\n'
' ... raise ValueError\n'
' ... except:\n'
- ' ... print(sys.exc_info())\n'
- ' ... print(sys.exc_info())\n'
+ ' ... print(repr(sys.exception()))\n'
+ ' ... print(repr(sys.exception()))\n'
' ...\n'
- " (<class 'TypeError'>, TypeError(), <traceback object at "
- '0x10efad080>)\n'
- " (<class 'ValueError'>, ValueError(), <traceback object at "
- '0x10efad040>)\n'
- " (<class 'TypeError'>, TypeError(), <traceback object at "
- '0x10efad080>)\n'
- ' >>> print(sys.exc_info())\n'
- ' (None, None, None)\n'
+ ' TypeError()\n'
+ ' ValueError()\n'
+ ' TypeError()\n'
+ ' >>> print(sys.exception())\n'
+ ' None\n'
+ '\n'
+ '\n'
+ '"except*" clause\n'
+ '================\n'
+ '\n'
+ 'The "except*" clause(s) are used for handling "ExceptionGroup"s. The\n'
+ 'exception type for matching is interpreted as in the case of '
+ '"except",\n'
+ 'but in the case of exception groups we can have partial matches when\n'
+ 'the type matches some of the exceptions in the group. This means '
+ 'that\n'
+ 'multiple "except*" clauses can execute, each handling part of the\n'
+ 'exception group. Each clause executes at most once and handles an\n'
+ 'exception group of all matching exceptions. Each exception in the\n'
+ 'group is handled by at most one "except*" clause, the first that\n'
+ 'matches it.\n'
+ '\n'
+ ' >>> try:\n'
+ ' ... raise ExceptionGroup("eg",\n'
+ ' ... [ValueError(1), TypeError(2), OSError(3), '
+ 'OSError(4)])\n'
+ ' ... except* TypeError as e:\n'
+ " ... print(f'caught {type(e)} with nested {e.exceptions}')\n"
+ ' ... except* OSError as e:\n'
+ " ... print(f'caught {type(e)} with nested {e.exceptions}')\n"
+ ' ...\n'
+ " caught <class 'ExceptionGroup'> with nested (TypeError(2),)\n"
+ " caught <class 'ExceptionGroup'> with nested (OSError(3), "
+ 'OSError(4))\n'
+ ' + Exception Group Traceback (most recent call last):\n'
+ ' | File "<stdin>", line 2, in <module>\n'
+ ' | ExceptionGroup: eg\n'
+ ' +-+---------------- 1 ----------------\n'
+ ' | ValueError: 1\n'
+ ' +------------------------------------\n'
+ '\n'
+ 'Any remaining exceptions that were not handled by any "except*" '
+ 'clause\n'
+ 'are re-raised at the end, combined into an exception group along '
+ 'with\n'
+ 'all exceptions that were raised from within "except*" clauses.\n'
+ '\n'
+ 'From version 3.11.4, when the entire "ExceptionGroup" is handled and\n'
+ 'only one exception is raised from an "except*" clause, this '
+ 'exception\n'
+ 'is no longer wrapped to form a new "ExceptionGroup".\n'
+ '\n'
+ 'If the raised exception is not an exception group and its type '
+ 'matches\n'
+ 'one of the "except*" clauses, it is caught and wrapped by an '
+ 'exception\n'
+ 'group with an empty message string.\n'
+ '\n'
+ ' >>> try:\n'
+ ' ... raise BlockingIOError\n'
+ ' ... except* BlockingIOError as e:\n'
+ ' ... print(repr(e))\n'
+ ' ...\n'
+ " ExceptionGroup('', (BlockingIOError()))\n"
+ '\n'
+ 'An "except*" clause must have a matching type, and this type cannot '
+ 'be\n'
+ 'a subclass of "BaseExceptionGroup". It is not possible to mix '
+ '"except"\n'
+ 'and "except*" in the same "try". "break", "continue" and "return"\n'
+ 'cannot appear in an "except*" clause.\n'
+ '\n'
+ '\n'
+ '"else" clause\n'
+ '=============\n'
'\n'
'The optional "else" clause is executed if the control flow leaves '
'the\n'
@@ -12551,6 +12815,10 @@
'"break" statement was executed. Exceptions in the "else" clause are\n'
'not handled by the preceding "except" clauses.\n'
'\n'
+ '\n'
+ '"finally" clause\n'
+ '================\n'
+ '\n'
'If "finally" is present, it specifies a ‘cleanup’ handler. The '
'"try"\n'
'clause is executed, including any "except" and "else" clauses. If '
@@ -12598,11 +12866,6 @@
' >>> foo()\n'
" 'finally'\n"
'\n'
- 'Additional information on exceptions can be found in section\n'
- 'Exceptions, and information on using the "raise" statement to '
- 'generate\n'
- 'exceptions may be found in section The raise statement.\n'
- '\n'
'Changed in version 3.8: Prior to Python 3.8, a "continue" statement\n'
'was illegal in the "finally" clause due to a problem with the\n'
'implementation.\n',
@@ -13483,6 +13746,7 @@
'\n'
' Special read-only attributes: "co_name" gives the function '
'name;\n'
+ ' "co_qualname" gives the fully qualified function name;\n'
' "co_argcount" is the total number of positional arguments\n'
' (including positional-only arguments and arguments with '
'default\n'
@@ -13543,6 +13807,54 @@
' "co_consts" is the documentation string of the function, or\n'
' "None" if undefined.\n'
'\n'
+ ' codeobject.co_positions()\n'
+ '\n'
+ ' Returns an iterable over the source code positions of '
+ 'each\n'
+ ' bytecode instruction in the code object.\n'
+ '\n'
+ ' The iterator returns tuples containing the "(start_line,\n'
+ ' end_line, start_column, end_column)". The *i-th* tuple\n'
+ ' corresponds to the position of the source code that '
+ 'compiled\n'
+ ' to the *i-th* instruction. Column information is '
+ '0-indexed\n'
+ ' utf-8 byte offsets on the given source line.\n'
+ '\n'
+ ' This positional information can be missing. A '
+ 'non-exhaustive\n'
+ ' lists of cases where this may happen:\n'
+ '\n'
+ ' * Running the interpreter with "-X" "no_debug_ranges".\n'
+ '\n'
+ ' * Loading a pyc file compiled while using "-X"\n'
+ ' "no_debug_ranges".\n'
+ '\n'
+ ' * Position tuples corresponding to artificial '
+ 'instructions.\n'
+ '\n'
+ ' * Line and column numbers that can’t be represented due '
+ 'to\n'
+ ' implementation specific limitations.\n'
+ '\n'
+ ' When this occurs, some or all of the tuple elements can '
+ 'be\n'
+ ' "None".\n'
+ '\n'
+ ' New in version 3.11.\n'
+ '\n'
+ ' Note:\n'
+ '\n'
+ ' This feature requires storing column positions in code\n'
+ ' objects which may result in a small increase of disk '
+ 'usage\n'
+ ' of compiled Python files or interpreter memory usage. '
+ 'To\n'
+ ' avoid storing the extra information and/or deactivate\n'
+ ' printing the extra traceback information, the "-X"\n'
+ ' "no_debug_ranges" command line flag or the\n'
+ ' "PYTHONNODEBUGRANGES" environment variable can be used.\n'
+ '\n'
' Frame objects\n'
' Frame objects represent execution frames. They may occur in\n'
' traceback objects (see below), and are also passed to '
diff --git a/common/py3-stdlib/random.py b/common/py3-stdlib/random.py
index 1310a2d..d07fffb 100644
--- a/common/py3-stdlib/random.py
+++ b/common/py3-stdlib/random.py
@@ -154,15 +154,11 @@
elif version == 2 and isinstance(a, (str, bytes, bytearray)):
if isinstance(a, str):
a = a.encode()
- a = int.from_bytes(a + _sha512(a).digest(), 'big')
+ a = int.from_bytes(a + _sha512(a).digest())
elif not isinstance(a, (type(None), int, float, str, bytes, bytearray)):
- _warn('Seeding based on hashing is deprecated\n'
- 'since Python 3.9 and will be removed in a subsequent '
- 'version. The only \n'
- 'supported seed types are: None, '
- 'int, float, str, bytes, and bytearray.',
- DeprecationWarning, 2)
+ raise TypeError('The only supported seed types are: None,\n'
+ 'int, float, str, bytes, and bytearray.')
super().seed(a)
self.gauss_next = None
@@ -237,10 +233,8 @@
break
def _randbelow_with_getrandbits(self, n):
- "Return a random int in the range [0,n). Returns 0 if n==0."
+ "Return a random int in the range [0,n). Defined for n > 0."
- if not n:
- return 0
getrandbits = self.getrandbits
k = n.bit_length() # don't use (n-1) here because n can be 1
r = getrandbits(k) # 0 <= r < 2**k
@@ -249,7 +243,7 @@
return r
def _randbelow_without_getrandbits(self, n, maxsize=1<<BPF):
- """Return a random int in the range [0,n). Returns 0 if n==0.
+ """Return a random int in the range [0,n). Defined for n > 0.
The implementation does not use getrandbits, but only random.
"""
@@ -260,8 +254,6 @@
"enough bits to choose from a population range this large.\n"
"To remove the range limitation, add a getrandbits() method.")
return _floor(random() * n)
- if n == 0:
- return 0
rem = maxsize % n
limit = (maxsize - rem) / maxsize # int(limit * maxsize) % n == 0
r = random()
@@ -290,10 +282,10 @@
## -------------------- integer methods -------------------
def randrange(self, start, stop=None, step=_ONE):
- """Choose a random item from range(start, stop[, step]).
+ """Choose a random item from range(stop) or range(start, stop[, step]).
- This fixes the problem with randint() which includes the
- endpoint; in Python this is usually not what you want.
+ Roughly equivalent to ``choice(range(start, stop, step))`` but
+ supports arbitrarily large ranges and is optimized for common cases.
"""
@@ -374,37 +366,24 @@
def choice(self, seq):
"""Choose a random element from a non-empty sequence."""
- # raises IndexError if seq is empty
+
+ # As an accommodation for NumPy, we don't use "if not seq"
+ # because bool(numpy.array()) raises a ValueError.
+ if not len(seq):
+ raise IndexError('Cannot choose from an empty sequence')
return seq[self._randbelow(len(seq))]
- def shuffle(self, x, random=None):
- """Shuffle list x in place, and return None.
+ def shuffle(self, x):
+ """Shuffle list x in place, and return None."""
- Optional argument random is a 0-argument function returning a
- random float in [0.0, 1.0); if it is the default None, the
- standard random.random will be used.
-
- """
-
- if random is None:
- randbelow = self._randbelow
- for i in reversed(range(1, len(x))):
- # pick an element in x[:i+1] with which to exchange x[i]
- j = randbelow(i + 1)
- x[i], x[j] = x[j], x[i]
- else:
- _warn('The *random* parameter to shuffle() has been deprecated\n'
- 'since Python 3.9 and will be removed in a subsequent '
- 'version.',
- DeprecationWarning, 2)
- floor = _floor
- for i in reversed(range(1, len(x))):
- # pick an element in x[:i+1] with which to exchange x[i]
- j = floor(random() * (i + 1))
- x[i], x[j] = x[j], x[i]
+ randbelow = self._randbelow
+ for i in reversed(range(1, len(x))):
+ # pick an element in x[:i+1] with which to exchange x[i]
+ j = randbelow(i + 1)
+ x[i], x[j] = x[j], x[i]
def sample(self, population, k, *, counts=None):
- """Chooses k unique random elements from a population sequence or set.
+ """Chooses k unique random elements from a population sequence.
Returns a new list containing elements from the population while
leaving the original population unchanged. The resulting list is
@@ -457,13 +436,8 @@
# causing them to eat more entropy than necessary.
if not isinstance(population, _Sequence):
- if isinstance(population, _Set):
- _warn('Sampling from a set deprecated\n'
- 'since Python 3.9 and will be removed in a subsequent version.',
- DeprecationWarning, 2)
- population = tuple(population)
- else:
- raise TypeError("Population must be a sequence. For dicts or sets, use sorted(d).")
+ raise TypeError("Population must be a sequence. "
+ "For dicts or sets, use sorted(d).")
n = len(population)
if counts is not None:
cum_counts = list(_accumulate(counts))
@@ -567,7 +541,7 @@
low, high = high, low
return low + (high - low) * _sqrt(u * c)
- def normalvariate(self, mu, sigma):
+ def normalvariate(self, mu=0.0, sigma=1.0):
"""Normal distribution.
mu is the mean, and sigma is the standard deviation.
@@ -588,7 +562,7 @@
break
return mu + z * sigma
- def gauss(self, mu, sigma):
+ def gauss(self, mu=0.0, sigma=1.0):
"""Gaussian distribution.
mu is the mean, and sigma is the standard deviation. This is
@@ -820,15 +794,15 @@
"""
def random(self):
- """Get the next random number in the range [0.0, 1.0)."""
- return (int.from_bytes(_urandom(7), 'big') >> 3) * RECIP_BPF
+ """Get the next random number in the range 0.0 <= X < 1.0."""
+ return (int.from_bytes(_urandom(7)) >> 3) * RECIP_BPF
def getrandbits(self, k):
"""getrandbits(k) -> x. Generates an int with k random bits."""
if k < 0:
raise ValueError('number of bits must be non-negative')
numbytes = (k + 7) // 8 # bits / 8 and rounded up
- x = int.from_bytes(_urandom(numbytes), 'big')
+ x = int.from_bytes(_urandom(numbytes))
return x >> (numbytes * 8 - k) # trim excess bits
def randbytes(self, n):
diff --git a/common/py3-stdlib/re.py b/common/py3-stdlib/re/__init__.py
similarity index 84%
rename from common/py3-stdlib/re.py
rename to common/py3-stdlib/re/__init__.py
index 1d82b50..d58c211 100644
--- a/common/py3-stdlib/re.py
+++ b/common/py3-stdlib/re/__init__.py
@@ -122,13 +122,8 @@
"""
import enum
-import sre_compile
-import sre_parse
+from . import _compiler, _parser
import functools
-try:
- import _locale
-except ImportError:
- _locale = None
# public symbols
@@ -137,49 +132,30 @@
"findall", "finditer", "compile", "purge", "template", "escape",
"error", "Pattern", "Match", "A", "I", "L", "M", "S", "X", "U",
"ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE",
- "UNICODE",
+ "UNICODE", "NOFLAG", "RegexFlag",
]
__version__ = "2.2.1"
-class RegexFlag(enum.IntFlag):
- ASCII = A = sre_compile.SRE_FLAG_ASCII # assume ascii "locale"
- IGNORECASE = I = sre_compile.SRE_FLAG_IGNORECASE # ignore case
- LOCALE = L = sre_compile.SRE_FLAG_LOCALE # assume current 8-bit locale
- UNICODE = U = sre_compile.SRE_FLAG_UNICODE # assume unicode "locale"
- MULTILINE = M = sre_compile.SRE_FLAG_MULTILINE # make anchors look for newline
- DOTALL = S = sre_compile.SRE_FLAG_DOTALL # make dot match newline
- VERBOSE = X = sre_compile.SRE_FLAG_VERBOSE # ignore whitespace and comments
+@enum.global_enum
+@enum._simple_enum(enum.IntFlag, boundary=enum.KEEP)
+class RegexFlag:
+ NOFLAG = 0
+ ASCII = A = _compiler.SRE_FLAG_ASCII # assume ascii "locale"
+ IGNORECASE = I = _compiler.SRE_FLAG_IGNORECASE # ignore case
+ LOCALE = L = _compiler.SRE_FLAG_LOCALE # assume current 8-bit locale
+ UNICODE = U = _compiler.SRE_FLAG_UNICODE # assume unicode "locale"
+ MULTILINE = M = _compiler.SRE_FLAG_MULTILINE # make anchors look for newline
+ DOTALL = S = _compiler.SRE_FLAG_DOTALL # make dot match newline
+ VERBOSE = X = _compiler.SRE_FLAG_VERBOSE # ignore whitespace and comments
# sre extensions (experimental, don't rely on these)
- TEMPLATE = T = sre_compile.SRE_FLAG_TEMPLATE # disable backtracking
- DEBUG = sre_compile.SRE_FLAG_DEBUG # dump pattern after compilation
-
- def __repr__(self):
- if self._name_ is not None:
- return f're.{self._name_}'
- value = self._value_
- members = []
- negative = value < 0
- if negative:
- value = ~value
- for m in self.__class__:
- if value & m._value_:
- value &= ~m._value_
- members.append(f're.{m._name_}')
- if value:
- members.append(hex(value))
- res = '|'.join(members)
- if negative:
- if len(members) > 1:
- res = f'~({res})'
- else:
- res = f'~{res}'
- return res
+ TEMPLATE = T = _compiler.SRE_FLAG_TEMPLATE # unknown purpose, deprecated
+ DEBUG = _compiler.SRE_FLAG_DEBUG # dump pattern after compilation
__str__ = object.__str__
-globals().update(RegexFlag.__members__)
+ _numeric_repr_ = hex
# sre exception
-error = sre_compile.error
+error = _compiler.error
# --------------------------------------------------------------------
# public interface
@@ -256,8 +232,16 @@
_compile_repl.cache_clear()
def template(pattern, flags=0):
- "Compile a template pattern, returning a Pattern object"
- return _compile(pattern, flags|T)
+ "Compile a template pattern, returning a Pattern object, deprecated"
+ import warnings
+ warnings.warn("The re.template() function is deprecated "
+ "as it is an undocumented function "
+ "without an obvious purpose. "
+ "Use re.compile() instead.",
+ DeprecationWarning)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning) # warn just once
+ return _compile(pattern, flags|T)
# SPECIAL_CHARS
# closing ')', '}' and ']'
@@ -276,8 +260,8 @@
pattern = str(pattern, 'latin1')
return pattern.translate(_special_chars_map).encode('latin1')
-Pattern = type(sre_compile.compile('', 0))
-Match = type(sre_compile.compile('', 0).match(''))
+Pattern = type(_compiler.compile('', 0))
+Match = type(_compiler.compile('', 0).match(''))
# --------------------------------------------------------------------
# internals
@@ -298,9 +282,16 @@
raise ValueError(
"cannot process flags argument with a compiled pattern")
return pattern
- if not sre_compile.isstring(pattern):
+ if not _compiler.isstring(pattern):
raise TypeError("first argument must be string or compiled pattern")
- p = sre_compile.compile(pattern, flags)
+ if flags & T:
+ import warnings
+ warnings.warn("The re.TEMPLATE/re.T flag is deprecated "
+ "as it is an undocumented flag "
+ "without an obvious purpose. "
+ "Don't use it.",
+ DeprecationWarning)
+ p = _compiler.compile(pattern, flags)
if not (flags & DEBUG):
if len(_cache) >= _MAXCACHE:
# Drop the oldest item
@@ -314,12 +305,12 @@
@functools.lru_cache(_MAXCACHE)
def _compile_repl(repl, pattern):
# internal: compile replacement pattern
- return sre_parse.parse_template(repl, pattern)
+ return _parser.parse_template(repl, pattern)
def _expand(pattern, match, template):
# internal: Match.expand implementation hook
- template = sre_parse.parse_template(template, pattern)
- return sre_parse.expand_template(template, match)
+ template = _parser.parse_template(template, pattern)
+ return _parser.expand_template(template, match)
def _subx(pattern, template):
# internal: Pattern.sub/subn implementation helper
@@ -328,7 +319,7 @@
# literal replacement
return template[1][0]
def filter(match, template=template):
- return sre_parse.expand_template(template, match)
+ return _parser.expand_template(template, match)
return filter
# register myself for pickling
@@ -345,22 +336,22 @@
class Scanner:
def __init__(self, lexicon, flags=0):
- from sre_constants import BRANCH, SUBPATTERN
+ from ._constants import BRANCH, SUBPATTERN
if isinstance(flags, RegexFlag):
flags = flags.value
self.lexicon = lexicon
# combine phrases into a compound pattern
p = []
- s = sre_parse.State()
+ s = _parser.State()
s.flags = flags
for phrase, action in lexicon:
gid = s.opengroup()
- p.append(sre_parse.SubPattern(s, [
- (SUBPATTERN, (gid, 0, 0, sre_parse.parse(phrase, flags))),
+ p.append(_parser.SubPattern(s, [
+ (SUBPATTERN, (gid, 0, 0, _parser.parse(phrase, flags))),
]))
s.closegroup(gid, p[-1])
- p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
- self.scanner = sre_compile.compile(p)
+ p = _parser.SubPattern(s, [(BRANCH, (None, p))])
+ self.scanner = _compiler.compile(p)
def scan(self, string):
result = []
append = result.append
diff --git a/common/py3-stdlib/re/_casefix.py b/common/py3-stdlib/re/_casefix.py
new file mode 100644
index 0000000..06507d0
--- /dev/null
+++ b/common/py3-stdlib/re/_casefix.py
@@ -0,0 +1,106 @@
+# Auto-generated by Tools/scripts/generate_re_casefix.py.
+
+# Maps the code of lowercased character to codes of different lowercased
+# characters which have the same uppercase.
+_EXTRA_CASES = {
+ # LATIN SMALL LETTER I: LATIN SMALL LETTER DOTLESS I
+ 0x0069: (0x0131,), # 'i': 'ı'
+ # LATIN SMALL LETTER S: LATIN SMALL LETTER LONG S
+ 0x0073: (0x017f,), # 's': 'ſ'
+ # MICRO SIGN: GREEK SMALL LETTER MU
+ 0x00b5: (0x03bc,), # 'µ': 'μ'
+ # LATIN SMALL LETTER DOTLESS I: LATIN SMALL LETTER I
+ 0x0131: (0x0069,), # 'ı': 'i'
+ # LATIN SMALL LETTER LONG S: LATIN SMALL LETTER S
+ 0x017f: (0x0073,), # 'ſ': 's'
+ # COMBINING GREEK YPOGEGRAMMENI: GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
+ 0x0345: (0x03b9, 0x1fbe), # '\u0345': 'ιι'
+ # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+ 0x0390: (0x1fd3,), # 'ΐ': 'ΐ'
+ # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+ 0x03b0: (0x1fe3,), # 'ΰ': 'ΰ'
+ # GREEK SMALL LETTER BETA: GREEK BETA SYMBOL
+ 0x03b2: (0x03d0,), # 'β': 'ϐ'
+ # GREEK SMALL LETTER EPSILON: GREEK LUNATE EPSILON SYMBOL
+ 0x03b5: (0x03f5,), # 'ε': 'ϵ'
+ # GREEK SMALL LETTER THETA: GREEK THETA SYMBOL
+ 0x03b8: (0x03d1,), # 'θ': 'ϑ'
+ # GREEK SMALL LETTER IOTA: COMBINING GREEK YPOGEGRAMMENI, GREEK PROSGEGRAMMENI
+ 0x03b9: (0x0345, 0x1fbe), # 'ι': '\u0345ι'
+ # GREEK SMALL LETTER KAPPA: GREEK KAPPA SYMBOL
+ 0x03ba: (0x03f0,), # 'κ': 'ϰ'
+ # GREEK SMALL LETTER MU: MICRO SIGN
+ 0x03bc: (0x00b5,), # 'μ': 'µ'
+ # GREEK SMALL LETTER PI: GREEK PI SYMBOL
+ 0x03c0: (0x03d6,), # 'π': 'ϖ'
+ # GREEK SMALL LETTER RHO: GREEK RHO SYMBOL
+ 0x03c1: (0x03f1,), # 'ρ': 'ϱ'
+ # GREEK SMALL LETTER FINAL SIGMA: GREEK SMALL LETTER SIGMA
+ 0x03c2: (0x03c3,), # 'ς': 'σ'
+ # GREEK SMALL LETTER SIGMA: GREEK SMALL LETTER FINAL SIGMA
+ 0x03c3: (0x03c2,), # 'σ': 'ς'
+ # GREEK SMALL LETTER PHI: GREEK PHI SYMBOL
+ 0x03c6: (0x03d5,), # 'φ': 'ϕ'
+ # GREEK BETA SYMBOL: GREEK SMALL LETTER BETA
+ 0x03d0: (0x03b2,), # 'ϐ': 'β'
+ # GREEK THETA SYMBOL: GREEK SMALL LETTER THETA
+ 0x03d1: (0x03b8,), # 'ϑ': 'θ'
+ # GREEK PHI SYMBOL: GREEK SMALL LETTER PHI
+ 0x03d5: (0x03c6,), # 'ϕ': 'φ'
+ # GREEK PI SYMBOL: GREEK SMALL LETTER PI
+ 0x03d6: (0x03c0,), # 'ϖ': 'π'
+ # GREEK KAPPA SYMBOL: GREEK SMALL LETTER KAPPA
+ 0x03f0: (0x03ba,), # 'ϰ': 'κ'
+ # GREEK RHO SYMBOL: GREEK SMALL LETTER RHO
+ 0x03f1: (0x03c1,), # 'ϱ': 'ρ'
+ # GREEK LUNATE EPSILON SYMBOL: GREEK SMALL LETTER EPSILON
+ 0x03f5: (0x03b5,), # 'ϵ': 'ε'
+ # CYRILLIC SMALL LETTER VE: CYRILLIC SMALL LETTER ROUNDED VE
+ 0x0432: (0x1c80,), # 'в': 'ᲀ'
+ # CYRILLIC SMALL LETTER DE: CYRILLIC SMALL LETTER LONG-LEGGED DE
+ 0x0434: (0x1c81,), # 'д': 'ᲁ'
+ # CYRILLIC SMALL LETTER O: CYRILLIC SMALL LETTER NARROW O
+ 0x043e: (0x1c82,), # 'о': 'ᲂ'
+ # CYRILLIC SMALL LETTER ES: CYRILLIC SMALL LETTER WIDE ES
+ 0x0441: (0x1c83,), # 'с': 'ᲃ'
+ # CYRILLIC SMALL LETTER TE: CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
+ 0x0442: (0x1c84, 0x1c85), # 'т': 'ᲄᲅ'
+ # CYRILLIC SMALL LETTER HARD SIGN: CYRILLIC SMALL LETTER TALL HARD SIGN
+ 0x044a: (0x1c86,), # 'ъ': 'ᲆ'
+ # CYRILLIC SMALL LETTER YAT: CYRILLIC SMALL LETTER TALL YAT
+ 0x0463: (0x1c87,), # 'ѣ': 'ᲇ'
+ # CYRILLIC SMALL LETTER ROUNDED VE: CYRILLIC SMALL LETTER VE
+ 0x1c80: (0x0432,), # 'ᲀ': 'в'
+ # CYRILLIC SMALL LETTER LONG-LEGGED DE: CYRILLIC SMALL LETTER DE
+ 0x1c81: (0x0434,), # 'ᲁ': 'д'
+ # CYRILLIC SMALL LETTER NARROW O: CYRILLIC SMALL LETTER O
+ 0x1c82: (0x043e,), # 'ᲂ': 'о'
+ # CYRILLIC SMALL LETTER WIDE ES: CYRILLIC SMALL LETTER ES
+ 0x1c83: (0x0441,), # 'ᲃ': 'с'
+ # CYRILLIC SMALL LETTER TALL TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
+ 0x1c84: (0x0442, 0x1c85), # 'ᲄ': 'тᲅ'
+ # CYRILLIC SMALL LETTER THREE-LEGGED TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE
+ 0x1c85: (0x0442, 0x1c84), # 'ᲅ': 'тᲄ'
+ # CYRILLIC SMALL LETTER TALL HARD SIGN: CYRILLIC SMALL LETTER HARD SIGN
+ 0x1c86: (0x044a,), # 'ᲆ': 'ъ'
+ # CYRILLIC SMALL LETTER TALL YAT: CYRILLIC SMALL LETTER YAT
+ 0x1c87: (0x0463,), # 'ᲇ': 'ѣ'
+ # CYRILLIC SMALL LETTER UNBLENDED UK: CYRILLIC SMALL LETTER MONOGRAPH UK
+ 0x1c88: (0xa64b,), # 'ᲈ': 'ꙋ'
+ # LATIN SMALL LETTER S WITH DOT ABOVE: LATIN SMALL LETTER LONG S WITH DOT ABOVE
+ 0x1e61: (0x1e9b,), # 'ṡ': 'ẛ'
+ # LATIN SMALL LETTER LONG S WITH DOT ABOVE: LATIN SMALL LETTER S WITH DOT ABOVE
+ 0x1e9b: (0x1e61,), # 'ẛ': 'ṡ'
+ # GREEK PROSGEGRAMMENI: COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA
+ 0x1fbe: (0x0345, 0x03b9), # 'ι': '\u0345ι'
+ # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+ 0x1fd3: (0x0390,), # 'ΐ': 'ΐ'
+ # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+ 0x1fe3: (0x03b0,), # 'ΰ': 'ΰ'
+ # CYRILLIC SMALL LETTER MONOGRAPH UK: CYRILLIC SMALL LETTER UNBLENDED UK
+ 0xa64b: (0x1c88,), # 'ꙋ': 'ᲈ'
+ # LATIN SMALL LIGATURE LONG S T: LATIN SMALL LIGATURE ST
+ 0xfb05: (0xfb06,), # 'ſt': 'st'
+ # LATIN SMALL LIGATURE ST: LATIN SMALL LIGATURE LONG S T
+ 0xfb06: (0xfb05,), # 'st': 'ſt'
+}
diff --git a/common/py3-stdlib/re/_compiler.py b/common/py3-stdlib/re/_compiler.py
new file mode 100644
index 0000000..d8e0d2f
--- /dev/null
+++ b/common/py3-stdlib/re/_compiler.py
@@ -0,0 +1,763 @@
+#
+# Secret Labs' Regular Expression Engine
+#
+# convert template to internal format
+#
+# Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved.
+#
+# See the __init__.py file for information on usage and redistribution.
+#
+
+"""Internal support module for sre"""
+
+import _sre
+from . import _parser
+from ._constants import *
+from ._casefix import _EXTRA_CASES
+
+assert _sre.MAGIC == MAGIC, "SRE module mismatch"
+
+_LITERAL_CODES = {LITERAL, NOT_LITERAL}
+_SUCCESS_CODES = {SUCCESS, FAILURE}
+_ASSERT_CODES = {ASSERT, ASSERT_NOT}
+_UNIT_CODES = _LITERAL_CODES | {ANY, IN}
+
+_REPEATING_CODES = {
+ MIN_REPEAT: (REPEAT, MIN_UNTIL, MIN_REPEAT_ONE),
+ MAX_REPEAT: (REPEAT, MAX_UNTIL, REPEAT_ONE),
+ POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE),
+}
+
+def _combine_flags(flags, add_flags, del_flags,
+ TYPE_FLAGS=_parser.TYPE_FLAGS):
+ if add_flags & TYPE_FLAGS:
+ flags &= ~TYPE_FLAGS
+ return (flags | add_flags) & ~del_flags
+
+def _compile(code, pattern, flags):
+ # internal: compile a (sub)pattern
+ emit = code.append
+ _len = len
+ LITERAL_CODES = _LITERAL_CODES
+ REPEATING_CODES = _REPEATING_CODES
+ SUCCESS_CODES = _SUCCESS_CODES
+ ASSERT_CODES = _ASSERT_CODES
+ iscased = None
+ tolower = None
+ fixes = None
+ if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE:
+ if flags & SRE_FLAG_UNICODE:
+ iscased = _sre.unicode_iscased
+ tolower = _sre.unicode_tolower
+ fixes = _EXTRA_CASES
+ else:
+ iscased = _sre.ascii_iscased
+ tolower = _sre.ascii_tolower
+ for op, av in pattern:
+ if op in LITERAL_CODES:
+ if not flags & SRE_FLAG_IGNORECASE:
+ emit(op)
+ emit(av)
+ elif flags & SRE_FLAG_LOCALE:
+ emit(OP_LOCALE_IGNORE[op])
+ emit(av)
+ elif not iscased(av):
+ emit(op)
+ emit(av)
+ else:
+ lo = tolower(av)
+ if not fixes: # ascii
+ emit(OP_IGNORE[op])
+ emit(lo)
+ elif lo not in fixes:
+ emit(OP_UNICODE_IGNORE[op])
+ emit(lo)
+ else:
+ emit(IN_UNI_IGNORE)
+ skip = _len(code); emit(0)
+ if op is NOT_LITERAL:
+ emit(NEGATE)
+ for k in (lo,) + fixes[lo]:
+ emit(LITERAL)
+ emit(k)
+ emit(FAILURE)
+ code[skip] = _len(code) - skip
+ elif op is IN:
+ charset, hascased = _optimize_charset(av, iscased, tolower, fixes)
+ if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE:
+ emit(IN_LOC_IGNORE)
+ elif not hascased:
+ emit(IN)
+ elif not fixes: # ascii
+ emit(IN_IGNORE)
+ else:
+ emit(IN_UNI_IGNORE)
+ skip = _len(code); emit(0)
+ _compile_charset(charset, flags, code)
+ code[skip] = _len(code) - skip
+ elif op is ANY:
+ if flags & SRE_FLAG_DOTALL:
+ emit(ANY_ALL)
+ else:
+ emit(ANY)
+ elif op in REPEATING_CODES:
+ if flags & SRE_FLAG_TEMPLATE:
+ raise error("internal: unsupported template operator %r" % (op,))
+ if _simple(av[2]):
+ emit(REPEATING_CODES[op][2])
+ skip = _len(code); emit(0)
+ emit(av[0])
+ emit(av[1])
+ _compile(code, av[2], flags)
+ emit(SUCCESS)
+ code[skip] = _len(code) - skip
+ else:
+ emit(REPEATING_CODES[op][0])
+ skip = _len(code); emit(0)
+ emit(av[0])
+ emit(av[1])
+ _compile(code, av[2], flags)
+ code[skip] = _len(code) - skip
+ emit(REPEATING_CODES[op][1])
+ elif op is SUBPATTERN:
+ group, add_flags, del_flags, p = av
+ if group:
+ emit(MARK)
+ emit((group-1)*2)
+ # _compile_info(code, p, _combine_flags(flags, add_flags, del_flags))
+ _compile(code, p, _combine_flags(flags, add_flags, del_flags))
+ if group:
+ emit(MARK)
+ emit((group-1)*2+1)
+ elif op is ATOMIC_GROUP:
+ # Atomic Groups are handled by starting with an Atomic
+ # Group op code, then putting in the atomic group pattern
+ # and finally a success op code to tell any repeat
+ # operations within the Atomic Group to stop eating and
+ # pop their stack if they reach it
+ emit(ATOMIC_GROUP)
+ skip = _len(code); emit(0)
+ _compile(code, av, flags)
+ emit(SUCCESS)
+ code[skip] = _len(code) - skip
+ elif op in SUCCESS_CODES:
+ emit(op)
+ elif op in ASSERT_CODES:
+ emit(op)
+ skip = _len(code); emit(0)
+ if av[0] >= 0:
+ emit(0) # look ahead
+ else:
+ lo, hi = av[1].getwidth()
+ if lo != hi:
+ raise error("look-behind requires fixed-width pattern")
+ emit(lo) # look behind
+ _compile(code, av[1], flags)
+ emit(SUCCESS)
+ code[skip] = _len(code) - skip
+ elif op is AT:
+ emit(op)
+ if flags & SRE_FLAG_MULTILINE:
+ av = AT_MULTILINE.get(av, av)
+ if flags & SRE_FLAG_LOCALE:
+ av = AT_LOCALE.get(av, av)
+ elif flags & SRE_FLAG_UNICODE:
+ av = AT_UNICODE.get(av, av)
+ emit(av)
+ elif op is BRANCH:
+ emit(op)
+ tail = []
+ tailappend = tail.append
+ for av in av[1]:
+ skip = _len(code); emit(0)
+ # _compile_info(code, av, flags)
+ _compile(code, av, flags)
+ emit(JUMP)
+ tailappend(_len(code)); emit(0)
+ code[skip] = _len(code) - skip
+ emit(FAILURE) # end of branch
+ for tail in tail:
+ code[tail] = _len(code) - tail
+ elif op is CATEGORY:
+ emit(op)
+ if flags & SRE_FLAG_LOCALE:
+ av = CH_LOCALE[av]
+ elif flags & SRE_FLAG_UNICODE:
+ av = CH_UNICODE[av]
+ emit(av)
+ elif op is GROUPREF:
+ if not flags & SRE_FLAG_IGNORECASE:
+ emit(op)
+ elif flags & SRE_FLAG_LOCALE:
+ emit(GROUPREF_LOC_IGNORE)
+ elif not fixes: # ascii
+ emit(GROUPREF_IGNORE)
+ else:
+ emit(GROUPREF_UNI_IGNORE)
+ emit(av-1)
+ elif op is GROUPREF_EXISTS:
+ emit(op)
+ emit(av[0]-1)
+ skipyes = _len(code); emit(0)
+ _compile(code, av[1], flags)
+ if av[2]:
+ emit(JUMP)
+ skipno = _len(code); emit(0)
+ code[skipyes] = _len(code) - skipyes + 1
+ _compile(code, av[2], flags)
+ code[skipno] = _len(code) - skipno
+ else:
+ code[skipyes] = _len(code) - skipyes + 1
+ else:
+ raise error("internal: unsupported operand type %r" % (op,))
+
+def _compile_charset(charset, flags, code):
+ # compile charset subprogram
+ emit = code.append
+ for op, av in charset:
+ emit(op)
+ if op is NEGATE:
+ pass
+ elif op is LITERAL:
+ emit(av)
+ elif op is RANGE or op is RANGE_UNI_IGNORE:
+ emit(av[0])
+ emit(av[1])
+ elif op is CHARSET:
+ code.extend(av)
+ elif op is BIGCHARSET:
+ code.extend(av)
+ elif op is CATEGORY:
+ if flags & SRE_FLAG_LOCALE:
+ emit(CH_LOCALE[av])
+ elif flags & SRE_FLAG_UNICODE:
+ emit(CH_UNICODE[av])
+ else:
+ emit(av)
+ else:
+ raise error("internal: unsupported set operator %r" % (op,))
+ emit(FAILURE)
+
+def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
+ # internal: optimize character set
+ out = []
+ tail = []
+ charmap = bytearray(256)
+ hascased = False
+ for op, av in charset:
+ while True:
+ try:
+ if op is LITERAL:
+ if fixup:
+ lo = fixup(av)
+ charmap[lo] = 1
+ if fixes and lo in fixes:
+ for k in fixes[lo]:
+ charmap[k] = 1
+ if not hascased and iscased(av):
+ hascased = True
+ else:
+ charmap[av] = 1
+ elif op is RANGE:
+ r = range(av[0], av[1]+1)
+ if fixup:
+ if fixes:
+ for i in map(fixup, r):
+ charmap[i] = 1
+ if i in fixes:
+ for k in fixes[i]:
+ charmap[k] = 1
+ else:
+ for i in map(fixup, r):
+ charmap[i] = 1
+ if not hascased:
+ hascased = any(map(iscased, r))
+ else:
+ for i in r:
+ charmap[i] = 1
+ elif op is NEGATE:
+ out.append((op, av))
+ else:
+ tail.append((op, av))
+ except IndexError:
+ if len(charmap) == 256:
+ # character set contains non-UCS1 character codes
+ charmap += b'\0' * 0xff00
+ continue
+ # Character set contains non-BMP character codes.
+ # For range, all BMP characters in the range are already
+ # proceeded.
+ if fixup:
+ hascased = True
+ # For now, IN_UNI_IGNORE+LITERAL and
+ # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP
+ # characters, because two characters (at least one of
+ # which is not in the BMP) match case-insensitively
+ # if and only if:
+ # 1) c1.lower() == c2.lower()
+ # 2) c1.lower() == c2 or c1.lower().upper() == c2
+ # Also, both c.lower() and c.lower().upper() are single
+ # characters for every non-BMP character.
+ if op is RANGE:
+ op = RANGE_UNI_IGNORE
+ tail.append((op, av))
+ break
+
+ # compress character map
+ runs = []
+ q = 0
+ while True:
+ p = charmap.find(1, q)
+ if p < 0:
+ break
+ if len(runs) >= 2:
+ runs = None
+ break
+ q = charmap.find(0, p)
+ if q < 0:
+ runs.append((p, len(charmap)))
+ break
+ runs.append((p, q))
+ if runs is not None:
+ # use literal/range
+ for p, q in runs:
+ if q - p == 1:
+ out.append((LITERAL, p))
+ else:
+ out.append((RANGE, (p, q - 1)))
+ out += tail
+ # if the case was changed or new representation is more compact
+ if hascased or len(out) < len(charset):
+ return out, hascased
+ # else original character set is good enough
+ return charset, hascased
+
+ # use bitmap
+ if len(charmap) == 256:
+ data = _mk_bitmap(charmap)
+ out.append((CHARSET, data))
+ out += tail
+ return out, hascased
+
+ # To represent a big charset, first a bitmap of all characters in the
+ # set is constructed. Then, this bitmap is sliced into chunks of 256
+ # characters, duplicate chunks are eliminated, and each chunk is
+ # given a number. In the compiled expression, the charset is
+ # represented by a 32-bit word sequence, consisting of one word for
+ # the number of different chunks, a sequence of 256 bytes (64 words)
+ # of chunk numbers indexed by their original chunk position, and a
+ # sequence of 256-bit chunks (8 words each).
+
+ # Compression is normally good: in a typical charset, large ranges of
+ # Unicode will be either completely excluded (e.g. if only cyrillic
+ # letters are to be matched), or completely included (e.g. if large
+ # subranges of Kanji match). These ranges will be represented by
+ # chunks of all one-bits or all zero-bits.
+
+ # Matching can be also done efficiently: the more significant byte of
+ # the Unicode character is an index into the chunk number, and the
+ # less significant byte is a bit index in the chunk (just like the
+ # CHARSET matching).
+
+ charmap = bytes(charmap) # should be hashable
+ comps = {}
+ mapping = bytearray(256)
+ block = 0
+ data = bytearray()
+ for i in range(0, 65536, 256):
+ chunk = charmap[i: i + 256]
+ if chunk in comps:
+ mapping[i // 256] = comps[chunk]
+ else:
+ mapping[i // 256] = comps[chunk] = block
+ block += 1
+ data += chunk
+ data = _mk_bitmap(data)
+ data[0:0] = [block] + _bytes_to_codes(mapping)
+ out.append((BIGCHARSET, data))
+ out += tail
+ return out, hascased
+
+_CODEBITS = _sre.CODESIZE * 8
+MAXCODE = (1 << _CODEBITS) - 1
+_BITS_TRANS = b'0' + b'1' * 255
+def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int):
+ s = bits.translate(_BITS_TRANS)[::-1]
+ return [_int(s[i - _CODEBITS: i], 2)
+ for i in range(len(s), 0, -_CODEBITS)]
+
+def _bytes_to_codes(b):
+ # Convert block indices to word array
+ a = memoryview(b).cast('I')
+ assert a.itemsize == _sre.CODESIZE
+ assert len(a) * a.itemsize == len(b)
+ return a.tolist()
+
+def _simple(p):
+ # check if this subpattern is a "simple" operator
+ if len(p) != 1:
+ return False
+ op, av = p[0]
+ if op is SUBPATTERN:
+ return av[0] is None and _simple(av[-1])
+ return op in _UNIT_CODES
+
+def _generate_overlap_table(prefix):
+ """
+ Generate an overlap table for the following prefix.
+ An overlap table is a table of the same size as the prefix which
+ informs about the potential self-overlap for each index in the prefix:
+ - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...]
+ - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with
+ prefix[0:k]
+ """
+ table = [0] * len(prefix)
+ for i in range(1, len(prefix)):
+ idx = table[i - 1]
+ while prefix[i] != prefix[idx]:
+ if idx == 0:
+ table[i] = 0
+ break
+ idx = table[idx - 1]
+ else:
+ table[i] = idx + 1
+ return table
+
+def _get_iscased(flags):
+ if not flags & SRE_FLAG_IGNORECASE:
+ return None
+ elif flags & SRE_FLAG_UNICODE:
+ return _sre.unicode_iscased
+ else:
+ return _sre.ascii_iscased
+
+def _get_literal_prefix(pattern, flags):
+ # look for literal prefix
+ prefix = []
+ prefixappend = prefix.append
+ prefix_skip = None
+ iscased = _get_iscased(flags)
+ for op, av in pattern.data:
+ if op is LITERAL:
+ if iscased and iscased(av):
+ break
+ prefixappend(av)
+ elif op is SUBPATTERN:
+ group, add_flags, del_flags, p = av
+ flags1 = _combine_flags(flags, add_flags, del_flags)
+ if flags1 & SRE_FLAG_IGNORECASE and flags1 & SRE_FLAG_LOCALE:
+ break
+ prefix1, prefix_skip1, got_all = _get_literal_prefix(p, flags1)
+ if prefix_skip is None:
+ if group is not None:
+ prefix_skip = len(prefix)
+ elif prefix_skip1 is not None:
+ prefix_skip = len(prefix) + prefix_skip1
+ prefix.extend(prefix1)
+ if not got_all:
+ break
+ else:
+ break
+ else:
+ return prefix, prefix_skip, True
+ return prefix, prefix_skip, False
+
+def _get_charset_prefix(pattern, flags):
+ while True:
+ if not pattern.data:
+ return None
+ op, av = pattern.data[0]
+ if op is not SUBPATTERN:
+ break
+ group, add_flags, del_flags, pattern = av
+ flags = _combine_flags(flags, add_flags, del_flags)
+ if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE:
+ return None
+
+ iscased = _get_iscased(flags)
+ if op is LITERAL:
+ if iscased and iscased(av):
+ return None
+ return [(op, av)]
+ elif op is BRANCH:
+ charset = []
+ charsetappend = charset.append
+ for p in av[1]:
+ if not p:
+ return None
+ op, av = p[0]
+ if op is LITERAL and not (iscased and iscased(av)):
+ charsetappend((op, av))
+ else:
+ return None
+ return charset
+ elif op is IN:
+ charset = av
+ if iscased:
+ for op, av in charset:
+ if op is LITERAL:
+ if iscased(av):
+ return None
+ elif op is RANGE:
+ if av[1] > 0xffff:
+ return None
+ if any(map(iscased, range(av[0], av[1]+1))):
+ return None
+ return charset
+ return None
+
+def _compile_info(code, pattern, flags):
+ # internal: compile an info block. in the current version,
+ # this contains min/max pattern width, and an optional literal
+ # prefix or a character map
+ lo, hi = pattern.getwidth()
+ if hi > MAXCODE:
+ hi = MAXCODE
+ if lo == 0:
+ code.extend([INFO, 4, 0, lo, hi])
+ return
+ # look for a literal prefix
+ prefix = []
+ prefix_skip = 0
+ charset = [] # not used
+ if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE):
+ # look for literal prefix
+ prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags)
+ # if no prefix, look for charset prefix
+ if not prefix:
+ charset = _get_charset_prefix(pattern, flags)
+## if prefix:
+## print("*** PREFIX", prefix, prefix_skip)
+## if charset:
+## print("*** CHARSET", charset)
+ # add an info block
+ emit = code.append
+ emit(INFO)
+ skip = len(code); emit(0)
+ # literal flag
+ mask = 0
+ if prefix:
+ mask = SRE_INFO_PREFIX
+ if prefix_skip is None and got_all:
+ mask = mask | SRE_INFO_LITERAL
+ elif charset:
+ mask = mask | SRE_INFO_CHARSET
+ emit(mask)
+ # pattern length
+ if lo < MAXCODE:
+ emit(lo)
+ else:
+ emit(MAXCODE)
+ prefix = prefix[:MAXCODE]
+ emit(min(hi, MAXCODE))
+ # add literal prefix
+ if prefix:
+ emit(len(prefix)) # length
+ if prefix_skip is None:
+ prefix_skip = len(prefix)
+ emit(prefix_skip) # skip
+ code.extend(prefix)
+ # generate overlap table
+ code.extend(_generate_overlap_table(prefix))
+ elif charset:
+ charset, hascased = _optimize_charset(charset)
+ assert not hascased
+ _compile_charset(charset, flags, code)
+ code[skip] = len(code) - skip
+
+def isstring(obj):
+ return isinstance(obj, (str, bytes))
+
+def _code(p, flags):
+
+ flags = p.state.flags | flags
+ code = []
+
+ # compile info block
+ _compile_info(code, p, flags)
+
+ # compile the pattern
+ _compile(code, p.data, flags)
+
+ code.append(SUCCESS)
+
+ return code
+
+def _hex_code(code):
+ return '[%s]' % ', '.join('%#0*x' % (_sre.CODESIZE*2+2, x) for x in code)
+
+def dis(code):
+ import sys
+
+ labels = set()
+ level = 0
+ offset_width = len(str(len(code) - 1))
+
+ def dis_(start, end):
+ def print_(*args, to=None):
+ if to is not None:
+ labels.add(to)
+ args += ('(to %d)' % (to,),)
+ print('%*d%s ' % (offset_width, start, ':' if start in labels else '.'),
+ end=' '*(level-1))
+ print(*args)
+
+ def print_2(*args):
+ print(end=' '*(offset_width + 2*level))
+ print(*args)
+
+ nonlocal level
+ level += 1
+ i = start
+ while i < end:
+ start = i
+ op = code[i]
+ i += 1
+ op = OPCODES[op]
+ if op in (SUCCESS, FAILURE, ANY, ANY_ALL,
+ MAX_UNTIL, MIN_UNTIL, NEGATE):
+ print_(op)
+ elif op in (LITERAL, NOT_LITERAL,
+ LITERAL_IGNORE, NOT_LITERAL_IGNORE,
+ LITERAL_UNI_IGNORE, NOT_LITERAL_UNI_IGNORE,
+ LITERAL_LOC_IGNORE, NOT_LITERAL_LOC_IGNORE):
+ arg = code[i]
+ i += 1
+ print_(op, '%#02x (%r)' % (arg, chr(arg)))
+ elif op is AT:
+ arg = code[i]
+ i += 1
+ arg = str(ATCODES[arg])
+ assert arg[:3] == 'AT_'
+ print_(op, arg[3:])
+ elif op is CATEGORY:
+ arg = code[i]
+ i += 1
+ arg = str(CHCODES[arg])
+ assert arg[:9] == 'CATEGORY_'
+ print_(op, arg[9:])
+ elif op in (IN, IN_IGNORE, IN_UNI_IGNORE, IN_LOC_IGNORE):
+ skip = code[i]
+ print_(op, skip, to=i+skip)
+ dis_(i+1, i+skip)
+ i += skip
+ elif op in (RANGE, RANGE_UNI_IGNORE):
+ lo, hi = code[i: i+2]
+ i += 2
+ print_(op, '%#02x %#02x (%r-%r)' % (lo, hi, chr(lo), chr(hi)))
+ elif op is CHARSET:
+ print_(op, _hex_code(code[i: i + 256//_CODEBITS]))
+ i += 256//_CODEBITS
+ elif op is BIGCHARSET:
+ arg = code[i]
+ i += 1
+ mapping = list(b''.join(x.to_bytes(_sre.CODESIZE, sys.byteorder)
+ for x in code[i: i + 256//_sre.CODESIZE]))
+ print_(op, arg, mapping)
+ i += 256//_sre.CODESIZE
+ level += 1
+ for j in range(arg):
+ print_2(_hex_code(code[i: i + 256//_CODEBITS]))
+ i += 256//_CODEBITS
+ level -= 1
+ elif op in (MARK, GROUPREF, GROUPREF_IGNORE, GROUPREF_UNI_IGNORE,
+ GROUPREF_LOC_IGNORE):
+ arg = code[i]
+ i += 1
+ print_(op, arg)
+ elif op is JUMP:
+ skip = code[i]
+ print_(op, skip, to=i+skip)
+ i += 1
+ elif op is BRANCH:
+ skip = code[i]
+ print_(op, skip, to=i+skip)
+ while skip:
+ dis_(i+1, i+skip)
+ i += skip
+ start = i
+ skip = code[i]
+ if skip:
+ print_('branch', skip, to=i+skip)
+ else:
+ print_(FAILURE)
+ i += 1
+ elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE,
+ POSSESSIVE_REPEAT, POSSESSIVE_REPEAT_ONE):
+ skip, min, max = code[i: i+3]
+ if max == MAXREPEAT:
+ max = 'MAXREPEAT'
+ print_(op, skip, min, max, to=i+skip)
+ dis_(i+3, i+skip)
+ i += skip
+ elif op is GROUPREF_EXISTS:
+ arg, skip = code[i: i+2]
+ print_(op, arg, skip, to=i+skip)
+ i += 2
+ elif op in (ASSERT, ASSERT_NOT):
+ skip, arg = code[i: i+2]
+ print_(op, skip, arg, to=i+skip)
+ dis_(i+2, i+skip)
+ i += skip
+ elif op is ATOMIC_GROUP:
+ skip = code[i]
+ print_(op, skip, to=i+skip)
+ dis_(i+1, i+skip)
+ i += skip
+ elif op is INFO:
+ skip, flags, min, max = code[i: i+4]
+ if max == MAXREPEAT:
+ max = 'MAXREPEAT'
+ print_(op, skip, bin(flags), min, max, to=i+skip)
+ start = i+4
+ if flags & SRE_INFO_PREFIX:
+ prefix_len, prefix_skip = code[i+4: i+6]
+ print_2(' prefix_skip', prefix_skip)
+ start = i + 6
+ prefix = code[start: start+prefix_len]
+ print_2(' prefix',
+ '[%s]' % ', '.join('%#02x' % x for x in prefix),
+ '(%r)' % ''.join(map(chr, prefix)))
+ start += prefix_len
+ print_2(' overlap', code[start: start+prefix_len])
+ start += prefix_len
+ if flags & SRE_INFO_CHARSET:
+ level += 1
+ print_2('in')
+ dis_(start, i+skip)
+ level -= 1
+ i += skip
+ else:
+ raise ValueError(op)
+
+ level -= 1
+
+ dis_(0, len(code))
+
+
+def compile(p, flags=0):
+ # internal: convert pattern list to internal format
+
+ if isstring(p):
+ pattern = p
+ p = _parser.parse(p, flags)
+ else:
+ pattern = None
+
+ code = _code(p, flags)
+
+ if flags & SRE_FLAG_DEBUG:
+ print()
+ dis(code)
+
+ # map in either direction
+ groupindex = p.state.groupdict
+ indexgroup = [None] * p.state.groups
+ for k, i in groupindex.items():
+ indexgroup[i] = k
+
+ return _sre.compile(
+ pattern, flags | p.state.flags, code,
+ p.state.groups-1,
+ groupindex, tuple(indexgroup)
+ )
diff --git a/common/py3-stdlib/re/_constants.py b/common/py3-stdlib/re/_constants.py
new file mode 100644
index 0000000..10ee14b
--- /dev/null
+++ b/common/py3-stdlib/re/_constants.py
@@ -0,0 +1,220 @@
+#
+# Secret Labs' Regular Expression Engine
+#
+# various symbols used by the regular expression engine.
+# run this script to update the _sre include files!
+#
+# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.
+#
+# See the __init__.py file for information on usage and redistribution.
+#
+
+"""Internal support module for sre"""
+
+# update when constants are added or removed
+
+MAGIC = 20220615
+
+from _sre import MAXREPEAT, MAXGROUPS
+
+# SRE standard exception (access as sre.error)
+# should this really be here?
+
+class error(Exception):
+ """Exception raised for invalid regular expressions.
+
+ Attributes:
+
+ msg: The unformatted error message
+ pattern: The regular expression pattern
+ pos: The index in the pattern where compilation failed (may be None)
+ lineno: The line corresponding to pos (may be None)
+ colno: The column corresponding to pos (may be None)
+ """
+
+ __module__ = 're'
+
+ def __init__(self, msg, pattern=None, pos=None):
+ self.msg = msg
+ self.pattern = pattern
+ self.pos = pos
+ if pattern is not None and pos is not None:
+ msg = '%s at position %d' % (msg, pos)
+ if isinstance(pattern, str):
+ newline = '\n'
+ else:
+ newline = b'\n'
+ self.lineno = pattern.count(newline, 0, pos) + 1
+ self.colno = pos - pattern.rfind(newline, 0, pos)
+ if newline in pattern:
+ msg = '%s (line %d, column %d)' % (msg, self.lineno, self.colno)
+ else:
+ self.lineno = self.colno = None
+ super().__init__(msg)
+
+
+class _NamedIntConstant(int):
+ def __new__(cls, value, name):
+ self = super(_NamedIntConstant, cls).__new__(cls, value)
+ self.name = name
+ return self
+
+ def __repr__(self):
+ return self.name
+
+ __reduce__ = None
+
+MAXREPEAT = _NamedIntConstant(MAXREPEAT, 'MAXREPEAT')
+
+def _makecodes(*names):
+ items = [_NamedIntConstant(i, name) for i, name in enumerate(names)]
+ globals().update({item.name: item for item in items})
+ return items
+
+# operators
+OPCODES = _makecodes(
+ # failure=0 success=1 (just because it looks better that way :-)
+ 'FAILURE', 'SUCCESS',
+
+ 'ANY', 'ANY_ALL',
+ 'ASSERT', 'ASSERT_NOT',
+ 'AT',
+ 'BRANCH',
+ 'CATEGORY',
+ 'CHARSET', 'BIGCHARSET',
+ 'GROUPREF', 'GROUPREF_EXISTS',
+ 'IN',
+ 'INFO',
+ 'JUMP',
+ 'LITERAL',
+ 'MARK',
+ 'MAX_UNTIL',
+ 'MIN_UNTIL',
+ 'NOT_LITERAL',
+ 'NEGATE',
+ 'RANGE',
+ 'REPEAT',
+ 'REPEAT_ONE',
+ 'SUBPATTERN',
+ 'MIN_REPEAT_ONE',
+ 'ATOMIC_GROUP',
+ 'POSSESSIVE_REPEAT',
+ 'POSSESSIVE_REPEAT_ONE',
+
+ 'GROUPREF_IGNORE',
+ 'IN_IGNORE',
+ 'LITERAL_IGNORE',
+ 'NOT_LITERAL_IGNORE',
+
+ 'GROUPREF_LOC_IGNORE',
+ 'IN_LOC_IGNORE',
+ 'LITERAL_LOC_IGNORE',
+ 'NOT_LITERAL_LOC_IGNORE',
+
+ 'GROUPREF_UNI_IGNORE',
+ 'IN_UNI_IGNORE',
+ 'LITERAL_UNI_IGNORE',
+ 'NOT_LITERAL_UNI_IGNORE',
+ 'RANGE_UNI_IGNORE',
+
+ # The following opcodes are only occurred in the parser output,
+ # but not in the compiled code.
+ 'MIN_REPEAT', 'MAX_REPEAT',
+)
+del OPCODES[-2:] # remove MIN_REPEAT and MAX_REPEAT
+
+# positions
+ATCODES = _makecodes(
+ 'AT_BEGINNING', 'AT_BEGINNING_LINE', 'AT_BEGINNING_STRING',
+ 'AT_BOUNDARY', 'AT_NON_BOUNDARY',
+ 'AT_END', 'AT_END_LINE', 'AT_END_STRING',
+
+ 'AT_LOC_BOUNDARY', 'AT_LOC_NON_BOUNDARY',
+
+ 'AT_UNI_BOUNDARY', 'AT_UNI_NON_BOUNDARY',
+)
+
+# categories
+CHCODES = _makecodes(
+ 'CATEGORY_DIGIT', 'CATEGORY_NOT_DIGIT',
+ 'CATEGORY_SPACE', 'CATEGORY_NOT_SPACE',
+ 'CATEGORY_WORD', 'CATEGORY_NOT_WORD',
+ 'CATEGORY_LINEBREAK', 'CATEGORY_NOT_LINEBREAK',
+
+ 'CATEGORY_LOC_WORD', 'CATEGORY_LOC_NOT_WORD',
+
+ 'CATEGORY_UNI_DIGIT', 'CATEGORY_UNI_NOT_DIGIT',
+ 'CATEGORY_UNI_SPACE', 'CATEGORY_UNI_NOT_SPACE',
+ 'CATEGORY_UNI_WORD', 'CATEGORY_UNI_NOT_WORD',
+ 'CATEGORY_UNI_LINEBREAK', 'CATEGORY_UNI_NOT_LINEBREAK',
+)
+
+
+# replacement operations for "ignore case" mode
+OP_IGNORE = {
+ LITERAL: LITERAL_IGNORE,
+ NOT_LITERAL: NOT_LITERAL_IGNORE,
+}
+
+OP_LOCALE_IGNORE = {
+ LITERAL: LITERAL_LOC_IGNORE,
+ NOT_LITERAL: NOT_LITERAL_LOC_IGNORE,
+}
+
+OP_UNICODE_IGNORE = {
+ LITERAL: LITERAL_UNI_IGNORE,
+ NOT_LITERAL: NOT_LITERAL_UNI_IGNORE,
+}
+
+AT_MULTILINE = {
+ AT_BEGINNING: AT_BEGINNING_LINE,
+ AT_END: AT_END_LINE
+}
+
+AT_LOCALE = {
+ AT_BOUNDARY: AT_LOC_BOUNDARY,
+ AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY
+}
+
+AT_UNICODE = {
+ AT_BOUNDARY: AT_UNI_BOUNDARY,
+ AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY
+}
+
+CH_LOCALE = {
+ CATEGORY_DIGIT: CATEGORY_DIGIT,
+ CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT,
+ CATEGORY_SPACE: CATEGORY_SPACE,
+ CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE,
+ CATEGORY_WORD: CATEGORY_LOC_WORD,
+ CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD,
+ CATEGORY_LINEBREAK: CATEGORY_LINEBREAK,
+ CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK
+}
+
+CH_UNICODE = {
+ CATEGORY_DIGIT: CATEGORY_UNI_DIGIT,
+ CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT,
+ CATEGORY_SPACE: CATEGORY_UNI_SPACE,
+ CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE,
+ CATEGORY_WORD: CATEGORY_UNI_WORD,
+ CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD,
+ CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK,
+ CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK
+}
+
+# flags
+SRE_FLAG_TEMPLATE = 1 # template mode (unknown purpose, deprecated)
+SRE_FLAG_IGNORECASE = 2 # case insensitive
+SRE_FLAG_LOCALE = 4 # honour system locale
+SRE_FLAG_MULTILINE = 8 # treat target as multiline string
+SRE_FLAG_DOTALL = 16 # treat target as a single string
+SRE_FLAG_UNICODE = 32 # use unicode "locale"
+SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments
+SRE_FLAG_DEBUG = 128 # debugging
+SRE_FLAG_ASCII = 256 # use ascii "locale"
+
+# flags for INFO primitive
+SRE_INFO_PREFIX = 1 # has prefix
+SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix)
+SRE_INFO_CHARSET = 4 # pattern starts with character from given set
diff --git a/common/py3-stdlib/re/_parser.py b/common/py3-stdlib/re/_parser.py
new file mode 100644
index 0000000..f747a03
--- /dev/null
+++ b/common/py3-stdlib/re/_parser.py
@@ -0,0 +1,1100 @@
+#
+# Secret Labs' Regular Expression Engine
+#
+# convert re-style regular expression to sre pattern
+#
+# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.
+#
+# See the __init__.py file for information on usage and redistribution.
+#
+
+"""Internal support module for sre"""
+
+# XXX: show string offset and offending character for all errors
+
+from ._constants import *
+
+SPECIAL_CHARS = ".\\[{()*+?^$|"
+REPEAT_CHARS = "*+?{"
+
+DIGITS = frozenset("0123456789")
+
+OCTDIGITS = frozenset("01234567")
+HEXDIGITS = frozenset("0123456789abcdefABCDEF")
+ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+WHITESPACE = frozenset(" \t\n\r\v\f")
+
+_REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT, POSSESSIVE_REPEAT})
+_UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY})
+
+ESCAPES = {
+ r"\a": (LITERAL, ord("\a")),
+ r"\b": (LITERAL, ord("\b")),
+ r"\f": (LITERAL, ord("\f")),
+ r"\n": (LITERAL, ord("\n")),
+ r"\r": (LITERAL, ord("\r")),
+ r"\t": (LITERAL, ord("\t")),
+ r"\v": (LITERAL, ord("\v")),
+ r"\\": (LITERAL, ord("\\"))
+}
+
+CATEGORIES = {
+ r"\A": (AT, AT_BEGINNING_STRING), # start of string
+ r"\b": (AT, AT_BOUNDARY),
+ r"\B": (AT, AT_NON_BOUNDARY),
+ r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]),
+ r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]),
+ r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]),
+ r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]),
+ r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]),
+ r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]),
+ r"\Z": (AT, AT_END_STRING), # end of string
+}
+
+FLAGS = {
+ # standard flags
+ "i": SRE_FLAG_IGNORECASE,
+ "L": SRE_FLAG_LOCALE,
+ "m": SRE_FLAG_MULTILINE,
+ "s": SRE_FLAG_DOTALL,
+ "x": SRE_FLAG_VERBOSE,
+ # extensions
+ "a": SRE_FLAG_ASCII,
+ "t": SRE_FLAG_TEMPLATE,
+ "u": SRE_FLAG_UNICODE,
+}
+
+TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE
+GLOBAL_FLAGS = SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE
+
+class State:
+ # keeps track of state for parsing
+ def __init__(self):
+ self.flags = 0
+ self.groupdict = {}
+ self.groupwidths = [None] # group 0
+ self.lookbehindgroups = None
+ self.grouprefpos = {}
+ @property
+ def groups(self):
+ return len(self.groupwidths)
+ def opengroup(self, name=None):
+ gid = self.groups
+ self.groupwidths.append(None)
+ if self.groups > MAXGROUPS:
+ raise error("too many groups")
+ if name is not None:
+ ogid = self.groupdict.get(name, None)
+ if ogid is not None:
+ raise error("redefinition of group name %r as group %d; "
+ "was group %d" % (name, gid, ogid))
+ self.groupdict[name] = gid
+ return gid
+ def closegroup(self, gid, p):
+ self.groupwidths[gid] = p.getwidth()
+ def checkgroup(self, gid):
+ return gid < self.groups and self.groupwidths[gid] is not None
+
+ def checklookbehindgroup(self, gid, source):
+ if self.lookbehindgroups is not None:
+ if not self.checkgroup(gid):
+ raise source.error('cannot refer to an open group')
+ if gid >= self.lookbehindgroups:
+ raise source.error('cannot refer to group defined in the same '
+ 'lookbehind subpattern')
+
+class SubPattern:
+ # a subpattern, in intermediate form
+ def __init__(self, state, data=None):
+ self.state = state
+ if data is None:
+ data = []
+ self.data = data
+ self.width = None
+
+ def dump(self, level=0):
+ nl = True
+ seqtypes = (tuple, list)
+ for op, av in self.data:
+ print(level*" " + str(op), end='')
+ if op is IN:
+ # member sublanguage
+ print()
+ for op, a in av:
+ print((level+1)*" " + str(op), a)
+ elif op is BRANCH:
+ print()
+ for i, a in enumerate(av[1]):
+ if i:
+ print(level*" " + "OR")
+ a.dump(level+1)
+ elif op is GROUPREF_EXISTS:
+ condgroup, item_yes, item_no = av
+ print('', condgroup)
+ item_yes.dump(level+1)
+ if item_no:
+ print(level*" " + "ELSE")
+ item_no.dump(level+1)
+ elif isinstance(av, seqtypes):
+ nl = False
+ for a in av:
+ if isinstance(a, SubPattern):
+ if not nl:
+ print()
+ a.dump(level+1)
+ nl = True
+ else:
+ if not nl:
+ print(' ', end='')
+ print(a, end='')
+ nl = False
+ if not nl:
+ print()
+ else:
+ print('', av)
+ def __repr__(self):
+ return repr(self.data)
+ def __len__(self):
+ return len(self.data)
+ def __delitem__(self, index):
+ del self.data[index]
+ def __getitem__(self, index):
+ if isinstance(index, slice):
+ return SubPattern(self.state, self.data[index])
+ return self.data[index]
+ def __setitem__(self, index, code):
+ self.data[index] = code
+ def insert(self, index, code):
+ self.data.insert(index, code)
+ def append(self, code):
+ self.data.append(code)
+ def getwidth(self):
+ # determine the width (min, max) for this subpattern
+ if self.width is not None:
+ return self.width
+ lo = hi = 0
+ for op, av in self.data:
+ if op is BRANCH:
+ i = MAXREPEAT - 1
+ j = 0
+ for av in av[1]:
+ l, h = av.getwidth()
+ i = min(i, l)
+ j = max(j, h)
+ lo = lo + i
+ hi = hi + j
+ elif op is ATOMIC_GROUP:
+ i, j = av.getwidth()
+ lo = lo + i
+ hi = hi + j
+ elif op is SUBPATTERN:
+ i, j = av[-1].getwidth()
+ lo = lo + i
+ hi = hi + j
+ elif op in _REPEATCODES:
+ i, j = av[2].getwidth()
+ lo = lo + i * av[0]
+ hi = hi + j * av[1]
+ elif op in _UNITCODES:
+ lo = lo + 1
+ hi = hi + 1
+ elif op is GROUPREF:
+ i, j = self.state.groupwidths[av]
+ lo = lo + i
+ hi = hi + j
+ elif op is GROUPREF_EXISTS:
+ i, j = av[1].getwidth()
+ if av[2] is not None:
+ l, h = av[2].getwidth()
+ i = min(i, l)
+ j = max(j, h)
+ else:
+ i = 0
+ lo = lo + i
+ hi = hi + j
+ elif op is SUCCESS:
+ break
+ self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT)
+ return self.width
+
+class Tokenizer:
+ def __init__(self, string):
+ self.istext = isinstance(string, str)
+ self.string = string
+ if not self.istext:
+ string = str(string, 'latin1')
+ self.decoded_string = string
+ self.index = 0
+ self.next = None
+ self.__next()
+ def __next(self):
+ index = self.index
+ try:
+ char = self.decoded_string[index]
+ except IndexError:
+ self.next = None
+ return
+ if char == "\\":
+ index += 1
+ try:
+ char += self.decoded_string[index]
+ except IndexError:
+ raise error("bad escape (end of pattern)",
+ self.string, len(self.string) - 1) from None
+ self.index = index + 1
+ self.next = char
+ def match(self, char):
+ if char == self.next:
+ self.__next()
+ return True
+ return False
+ def get(self):
+ this = self.next
+ self.__next()
+ return this
+ def getwhile(self, n, charset):
+ result = ''
+ for _ in range(n):
+ c = self.next
+ if c not in charset:
+ break
+ result += c
+ self.__next()
+ return result
+ def getuntil(self, terminator, name):
+ result = ''
+ while True:
+ c = self.next
+ self.__next()
+ if c is None:
+ if not result:
+ raise self.error("missing " + name)
+ raise self.error("missing %s, unterminated name" % terminator,
+ len(result))
+ if c == terminator:
+ if not result:
+ raise self.error("missing " + name, 1)
+ break
+ result += c
+ return result
+ @property
+ def pos(self):
+ return self.index - len(self.next or '')
+ def tell(self):
+ return self.index - len(self.next or '')
+ def seek(self, index):
+ self.index = index
+ self.__next()
+
+ def error(self, msg, offset=0):
+ if not self.istext:
+ msg = msg.encode('ascii', 'backslashreplace').decode('ascii')
+ return error(msg, self.string, self.tell() - offset)
+
+ def checkgroupname(self, name, offset, nested):
+ if not name.isidentifier():
+ msg = "bad character in group name %r" % name
+ raise self.error(msg, len(name) + offset)
+ if not (self.istext or name.isascii()):
+ import warnings
+ warnings.warn(
+ "bad character in group name %a at position %d" %
+ (name, self.tell() - len(name) - offset),
+ DeprecationWarning, stacklevel=nested + 7
+ )
+
+def _class_escape(source, escape):
+ # handle escape code inside character class
+ code = ESCAPES.get(escape)
+ if code:
+ return code
+ code = CATEGORIES.get(escape)
+ if code and code[0] is IN:
+ return code
+ try:
+ c = escape[1:2]
+ if c == "x":
+ # hexadecimal escape (exactly two digits)
+ escape += source.getwhile(2, HEXDIGITS)
+ if len(escape) != 4:
+ raise source.error("incomplete escape %s" % escape, len(escape))
+ return LITERAL, int(escape[2:], 16)
+ elif c == "u" and source.istext:
+ # unicode escape (exactly four digits)
+ escape += source.getwhile(4, HEXDIGITS)
+ if len(escape) != 6:
+ raise source.error("incomplete escape %s" % escape, len(escape))
+ return LITERAL, int(escape[2:], 16)
+ elif c == "U" and source.istext:
+ # unicode escape (exactly eight digits)
+ escape += source.getwhile(8, HEXDIGITS)
+ if len(escape) != 10:
+ raise source.error("incomplete escape %s" % escape, len(escape))
+ c = int(escape[2:], 16)
+ chr(c) # raise ValueError for invalid code
+ return LITERAL, c
+ elif c == "N" and source.istext:
+ import unicodedata
+ # named unicode escape e.g. \N{EM DASH}
+ if not source.match('{'):
+ raise source.error("missing {")
+ charname = source.getuntil('}', 'character name')
+ try:
+ c = ord(unicodedata.lookup(charname))
+ except (KeyError, TypeError):
+ raise source.error("undefined character name %r" % charname,
+ len(charname) + len(r'\N{}')) from None
+ return LITERAL, c
+ elif c in OCTDIGITS:
+ # octal escape (up to three digits)
+ escape += source.getwhile(2, OCTDIGITS)
+ c = int(escape[1:], 8)
+ if c > 0o377:
+ raise source.error('octal escape value %s outside of '
+ 'range 0-0o377' % escape, len(escape))
+ return LITERAL, c
+ elif c in DIGITS:
+ raise ValueError
+ if len(escape) == 2:
+ if c in ASCIILETTERS:
+ raise source.error('bad escape %s' % escape, len(escape))
+ return LITERAL, ord(escape[1])
+ except ValueError:
+ pass
+ raise source.error("bad escape %s" % escape, len(escape))
+
+def _escape(source, escape, state):
+ # handle escape code in expression
+ code = CATEGORIES.get(escape)
+ if code:
+ return code
+ code = ESCAPES.get(escape)
+ if code:
+ return code
+ try:
+ c = escape[1:2]
+ if c == "x":
+ # hexadecimal escape
+ escape += source.getwhile(2, HEXDIGITS)
+ if len(escape) != 4:
+ raise source.error("incomplete escape %s" % escape, len(escape))
+ return LITERAL, int(escape[2:], 16)
+ elif c == "u" and source.istext:
+ # unicode escape (exactly four digits)
+ escape += source.getwhile(4, HEXDIGITS)
+ if len(escape) != 6:
+ raise source.error("incomplete escape %s" % escape, len(escape))
+ return LITERAL, int(escape[2:], 16)
+ elif c == "U" and source.istext:
+ # unicode escape (exactly eight digits)
+ escape += source.getwhile(8, HEXDIGITS)
+ if len(escape) != 10:
+ raise source.error("incomplete escape %s" % escape, len(escape))
+ c = int(escape[2:], 16)
+ chr(c) # raise ValueError for invalid code
+ return LITERAL, c
+ elif c == "N" and source.istext:
+ import unicodedata
+ # named unicode escape e.g. \N{EM DASH}
+ if not source.match('{'):
+ raise source.error("missing {")
+ charname = source.getuntil('}', 'character name')
+ try:
+ c = ord(unicodedata.lookup(charname))
+ except (KeyError, TypeError):
+ raise source.error("undefined character name %r" % charname,
+ len(charname) + len(r'\N{}')) from None
+ return LITERAL, c
+ elif c == "0":
+ # octal escape
+ escape += source.getwhile(2, OCTDIGITS)
+ return LITERAL, int(escape[1:], 8)
+ elif c in DIGITS:
+ # octal escape *or* decimal group reference (sigh)
+ if source.next in DIGITS:
+ escape += source.get()
+ if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and
+ source.next in OCTDIGITS):
+ # got three octal digits; this is an octal escape
+ escape += source.get()
+ c = int(escape[1:], 8)
+ if c > 0o377:
+ raise source.error('octal escape value %s outside of '
+ 'range 0-0o377' % escape,
+ len(escape))
+ return LITERAL, c
+ # not an octal escape, so this is a group reference
+ group = int(escape[1:])
+ if group < state.groups:
+ if not state.checkgroup(group):
+ raise source.error("cannot refer to an open group",
+ len(escape))
+ state.checklookbehindgroup(group, source)
+ return GROUPREF, group
+ raise source.error("invalid group reference %d" % group, len(escape) - 1)
+ if len(escape) == 2:
+ if c in ASCIILETTERS:
+ raise source.error("bad escape %s" % escape, len(escape))
+ return LITERAL, ord(escape[1])
+ except ValueError:
+ pass
+ raise source.error("bad escape %s" % escape, len(escape))
+
+def _uniq(items):
+ return list(dict.fromkeys(items))
+
+def _parse_sub(source, state, verbose, nested):
+ # parse an alternation: a|b|c
+
+ items = []
+ itemsappend = items.append
+ sourcematch = source.match
+ start = source.tell()
+ while True:
+ itemsappend(_parse(source, state, verbose, nested + 1,
+ not nested and not items))
+ if not sourcematch("|"):
+ break
+ if not nested:
+ verbose = state.flags & SRE_FLAG_VERBOSE
+
+ if len(items) == 1:
+ return items[0]
+
+ subpattern = SubPattern(state)
+
+ # check if all items share a common prefix
+ while True:
+ prefix = None
+ for item in items:
+ if not item:
+ break
+ if prefix is None:
+ prefix = item[0]
+ elif item[0] != prefix:
+ break
+ else:
+ # all subitems start with a common "prefix".
+ # move it out of the branch
+ for item in items:
+ del item[0]
+ subpattern.append(prefix)
+ continue # check next one
+ break
+
+ # check if the branch can be replaced by a character set
+ set = []
+ for item in items:
+ if len(item) != 1:
+ break
+ op, av = item[0]
+ if op is LITERAL:
+ set.append((op, av))
+ elif op is IN and av[0][0] is not NEGATE:
+ set.extend(av)
+ else:
+ break
+ else:
+ # we can store this as a character set instead of a
+ # branch (the compiler may optimize this even more)
+ subpattern.append((IN, _uniq(set)))
+ return subpattern
+
+ subpattern.append((BRANCH, (None, items)))
+ return subpattern
+
+def _parse(source, state, verbose, nested, first=False):
+ # parse a simple pattern
+ subpattern = SubPattern(state)
+
+ # precompute constants into local variables
+ subpatternappend = subpattern.append
+ sourceget = source.get
+ sourcematch = source.match
+ _len = len
+ _ord = ord
+
+ while True:
+
+ this = source.next
+ if this is None:
+ break # end of pattern
+ if this in "|)":
+ break # end of subpattern
+ sourceget()
+
+ if verbose:
+ # skip whitespace and comments
+ if this in WHITESPACE:
+ continue
+ if this == "#":
+ while True:
+ this = sourceget()
+ if this is None or this == "\n":
+ break
+ continue
+
+ if this[0] == "\\":
+ code = _escape(source, this, state)
+ subpatternappend(code)
+
+ elif this not in SPECIAL_CHARS:
+ subpatternappend((LITERAL, _ord(this)))
+
+ elif this == "[":
+ here = source.tell() - 1
+ # character set
+ set = []
+ setappend = set.append
+## if sourcematch(":"):
+## pass # handle character classes
+ if source.next == '[':
+ import warnings
+ warnings.warn(
+ 'Possible nested set at position %d' % source.tell(),
+ FutureWarning, stacklevel=nested + 6
+ )
+ negate = sourcematch("^")
+ # check remaining characters
+ while True:
+ this = sourceget()
+ if this is None:
+ raise source.error("unterminated character set",
+ source.tell() - here)
+ if this == "]" and set:
+ break
+ elif this[0] == "\\":
+ code1 = _class_escape(source, this)
+ else:
+ if set and this in '-&~|' and source.next == this:
+ import warnings
+ warnings.warn(
+ 'Possible set %s at position %d' % (
+ 'difference' if this == '-' else
+ 'intersection' if this == '&' else
+ 'symmetric difference' if this == '~' else
+ 'union',
+ source.tell() - 1),
+ FutureWarning, stacklevel=nested + 6
+ )
+ code1 = LITERAL, _ord(this)
+ if sourcematch("-"):
+ # potential range
+ that = sourceget()
+ if that is None:
+ raise source.error("unterminated character set",
+ source.tell() - here)
+ if that == "]":
+ if code1[0] is IN:
+ code1 = code1[1][0]
+ setappend(code1)
+ setappend((LITERAL, _ord("-")))
+ break
+ if that[0] == "\\":
+ code2 = _class_escape(source, that)
+ else:
+ if that == '-':
+ import warnings
+ warnings.warn(
+ 'Possible set difference at position %d' % (
+ source.tell() - 2),
+ FutureWarning, stacklevel=nested + 6
+ )
+ code2 = LITERAL, _ord(that)
+ if code1[0] != LITERAL or code2[0] != LITERAL:
+ msg = "bad character range %s-%s" % (this, that)
+ raise source.error(msg, len(this) + 1 + len(that))
+ lo = code1[1]
+ hi = code2[1]
+ if hi < lo:
+ msg = "bad character range %s-%s" % (this, that)
+ raise source.error(msg, len(this) + 1 + len(that))
+ setappend((RANGE, (lo, hi)))
+ else:
+ if code1[0] is IN:
+ code1 = code1[1][0]
+ setappend(code1)
+
+ set = _uniq(set)
+ # XXX: <fl> should move set optimization to compiler!
+ if _len(set) == 1 and set[0][0] is LITERAL:
+ # optimization
+ if negate:
+ subpatternappend((NOT_LITERAL, set[0][1]))
+ else:
+ subpatternappend(set[0])
+ else:
+ if negate:
+ set.insert(0, (NEGATE, None))
+ # charmap optimization can't be added here because
+ # global flags still are not known
+ subpatternappend((IN, set))
+
+ elif this in REPEAT_CHARS:
+ # repeat previous item
+ here = source.tell()
+ if this == "?":
+ min, max = 0, 1
+ elif this == "*":
+ min, max = 0, MAXREPEAT
+
+ elif this == "+":
+ min, max = 1, MAXREPEAT
+ elif this == "{":
+ if source.next == "}":
+ subpatternappend((LITERAL, _ord(this)))
+ continue
+
+ min, max = 0, MAXREPEAT
+ lo = hi = ""
+ while source.next in DIGITS:
+ lo += sourceget()
+ if sourcematch(","):
+ while source.next in DIGITS:
+ hi += sourceget()
+ else:
+ hi = lo
+ if not sourcematch("}"):
+ subpatternappend((LITERAL, _ord(this)))
+ source.seek(here)
+ continue
+
+ if lo:
+ min = int(lo)
+ if min >= MAXREPEAT:
+ raise OverflowError("the repetition number is too large")
+ if hi:
+ max = int(hi)
+ if max >= MAXREPEAT:
+ raise OverflowError("the repetition number is too large")
+ if max < min:
+ raise source.error("min repeat greater than max repeat",
+ source.tell() - here)
+ else:
+ raise AssertionError("unsupported quantifier %r" % (char,))
+ # figure out which item to repeat
+ if subpattern:
+ item = subpattern[-1:]
+ else:
+ item = None
+ if not item or item[0][0] is AT:
+ raise source.error("nothing to repeat",
+ source.tell() - here + len(this))
+ if item[0][0] in _REPEATCODES:
+ raise source.error("multiple repeat",
+ source.tell() - here + len(this))
+ if item[0][0] is SUBPATTERN:
+ group, add_flags, del_flags, p = item[0][1]
+ if group is None and not add_flags and not del_flags:
+ item = p
+ if sourcematch("?"):
+ # Non-Greedy Match
+ subpattern[-1] = (MIN_REPEAT, (min, max, item))
+ elif sourcematch("+"):
+ # Possessive Match (Always Greedy)
+ subpattern[-1] = (POSSESSIVE_REPEAT, (min, max, item))
+ else:
+ # Greedy Match
+ subpattern[-1] = (MAX_REPEAT, (min, max, item))
+
+ elif this == ".":
+ subpatternappend((ANY, None))
+
+ elif this == "(":
+ start = source.tell() - 1
+ capture = True
+ atomic = False
+ name = None
+ add_flags = 0
+ del_flags = 0
+ if sourcematch("?"):
+ # options
+ char = sourceget()
+ if char is None:
+ raise source.error("unexpected end of pattern")
+ if char == "P":
+ # python extensions
+ if sourcematch("<"):
+ # named group: skip forward to end of name
+ name = source.getuntil(">", "group name")
+ source.checkgroupname(name, 1, nested)
+ elif sourcematch("="):
+ # named backreference
+ name = source.getuntil(")", "group name")
+ source.checkgroupname(name, 1, nested)
+ gid = state.groupdict.get(name)
+ if gid is None:
+ msg = "unknown group name %r" % name
+ raise source.error(msg, len(name) + 1)
+ if not state.checkgroup(gid):
+ raise source.error("cannot refer to an open group",
+ len(name) + 1)
+ state.checklookbehindgroup(gid, source)
+ subpatternappend((GROUPREF, gid))
+ continue
+
+ else:
+ char = sourceget()
+ if char is None:
+ raise source.error("unexpected end of pattern")
+ raise source.error("unknown extension ?P" + char,
+ len(char) + 2)
+ elif char == ":":
+ # non-capturing group
+ capture = False
+ elif char == "#":
+ # comment
+ while True:
+ if source.next is None:
+ raise source.error("missing ), unterminated comment",
+ source.tell() - start)
+ if sourceget() == ")":
+ break
+ continue
+
+ elif char in "=!<":
+ # lookahead assertions
+ dir = 1
+ if char == "<":
+ char = sourceget()
+ if char is None:
+ raise source.error("unexpected end of pattern")
+ if char not in "=!":
+ raise source.error("unknown extension ?<" + char,
+ len(char) + 2)
+ dir = -1 # lookbehind
+ lookbehindgroups = state.lookbehindgroups
+ if lookbehindgroups is None:
+ state.lookbehindgroups = state.groups
+ p = _parse_sub(source, state, verbose, nested + 1)
+ if dir < 0:
+ if lookbehindgroups is None:
+ state.lookbehindgroups = None
+ if not sourcematch(")"):
+ raise source.error("missing ), unterminated subpattern",
+ source.tell() - start)
+ if char == "=":
+ subpatternappend((ASSERT, (dir, p)))
+ else:
+ subpatternappend((ASSERT_NOT, (dir, p)))
+ continue
+
+ elif char == "(":
+ # conditional backreference group
+ condname = source.getuntil(")", "group name")
+ if condname.isidentifier():
+ source.checkgroupname(condname, 1, nested)
+ condgroup = state.groupdict.get(condname)
+ if condgroup is None:
+ msg = "unknown group name %r" % condname
+ raise source.error(msg, len(condname) + 1)
+ else:
+ try:
+ condgroup = int(condname)
+ if condgroup < 0:
+ raise ValueError
+ except ValueError:
+ msg = "bad character in group name %r" % condname
+ raise source.error(msg, len(condname) + 1) from None
+ if not condgroup:
+ raise source.error("bad group number",
+ len(condname) + 1)
+ if condgroup >= MAXGROUPS:
+ msg = "invalid group reference %d" % condgroup
+ raise source.error(msg, len(condname) + 1)
+ if condgroup not in state.grouprefpos:
+ state.grouprefpos[condgroup] = (
+ source.tell() - len(condname) - 1
+ )
+ if not (condname.isdecimal() and condname.isascii()):
+ import warnings
+ warnings.warn(
+ "bad character in group name %s at position %d" %
+ (repr(condname) if source.istext else ascii(condname),
+ source.tell() - len(condname) - 1),
+ DeprecationWarning, stacklevel=nested + 6
+ )
+ state.checklookbehindgroup(condgroup, source)
+ item_yes = _parse(source, state, verbose, nested + 1)
+ if source.match("|"):
+ item_no = _parse(source, state, verbose, nested + 1)
+ if source.next == "|":
+ raise source.error("conditional backref with more than two branches")
+ else:
+ item_no = None
+ if not source.match(")"):
+ raise source.error("missing ), unterminated subpattern",
+ source.tell() - start)
+ subpatternappend((GROUPREF_EXISTS, (condgroup, item_yes, item_no)))
+ continue
+
+ elif char == ">":
+ # non-capturing, atomic group
+ capture = False
+ atomic = True
+ elif char in FLAGS or char == "-":
+ # flags
+ flags = _parse_flags(source, state, char)
+ if flags is None: # global flags
+ if not first or subpattern:
+ raise source.error('global flags not at the start '
+ 'of the expression',
+ source.tell() - start)
+ verbose = state.flags & SRE_FLAG_VERBOSE
+ continue
+
+ add_flags, del_flags = flags
+ capture = False
+ else:
+ raise source.error("unknown extension ?" + char,
+ len(char) + 1)
+
+ # parse group contents
+ if capture:
+ try:
+ group = state.opengroup(name)
+ except error as err:
+ raise source.error(err.msg, len(name) + 1) from None
+ else:
+ group = None
+ sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and
+ not (del_flags & SRE_FLAG_VERBOSE))
+ p = _parse_sub(source, state, sub_verbose, nested + 1)
+ if not source.match(")"):
+ raise source.error("missing ), unterminated subpattern",
+ source.tell() - start)
+ if group is not None:
+ state.closegroup(group, p)
+ if atomic:
+ assert group is None
+ subpatternappend((ATOMIC_GROUP, p))
+ else:
+ subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p)))
+
+ elif this == "^":
+ subpatternappend((AT, AT_BEGINNING))
+
+ elif this == "$":
+ subpatternappend((AT, AT_END))
+
+ else:
+ raise AssertionError("unsupported special character %r" % (char,))
+
+ # unpack non-capturing groups
+ for i in range(len(subpattern))[::-1]:
+ op, av = subpattern[i]
+ if op is SUBPATTERN:
+ group, add_flags, del_flags, p = av
+ if group is None and not add_flags and not del_flags:
+ subpattern[i: i+1] = p
+
+ return subpattern
+
+def _parse_flags(source, state, char):
+ sourceget = source.get
+ add_flags = 0
+ del_flags = 0
+ if char != "-":
+ while True:
+ flag = FLAGS[char]
+ if source.istext:
+ if char == 'L':
+ msg = "bad inline flags: cannot use 'L' flag with a str pattern"
+ raise source.error(msg)
+ else:
+ if char == 'u':
+ msg = "bad inline flags: cannot use 'u' flag with a bytes pattern"
+ raise source.error(msg)
+ add_flags |= flag
+ if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag:
+ msg = "bad inline flags: flags 'a', 'u' and 'L' are incompatible"
+ raise source.error(msg)
+ char = sourceget()
+ if char is None:
+ raise source.error("missing -, : or )")
+ if char in ")-:":
+ break
+ if char not in FLAGS:
+ msg = "unknown flag" if char.isalpha() else "missing -, : or )"
+ raise source.error(msg, len(char))
+ if char == ")":
+ state.flags |= add_flags
+ return None
+ if add_flags & GLOBAL_FLAGS:
+ raise source.error("bad inline flags: cannot turn on global flag", 1)
+ if char == "-":
+ char = sourceget()
+ if char is None:
+ raise source.error("missing flag")
+ if char not in FLAGS:
+ msg = "unknown flag" if char.isalpha() else "missing flag"
+ raise source.error(msg, len(char))
+ while True:
+ flag = FLAGS[char]
+ if flag & TYPE_FLAGS:
+ msg = "bad inline flags: cannot turn off flags 'a', 'u' and 'L'"
+ raise source.error(msg)
+ del_flags |= flag
+ char = sourceget()
+ if char is None:
+ raise source.error("missing :")
+ if char == ":":
+ break
+ if char not in FLAGS:
+ msg = "unknown flag" if char.isalpha() else "missing :"
+ raise source.error(msg, len(char))
+ assert char == ":"
+ if del_flags & GLOBAL_FLAGS:
+ raise source.error("bad inline flags: cannot turn off global flag", 1)
+ if add_flags & del_flags:
+ raise source.error("bad inline flags: flag turned on and off", 1)
+ return add_flags, del_flags
+
+def fix_flags(src, flags):
+ # Check and fix flags according to the type of pattern (str or bytes)
+ if isinstance(src, str):
+ if flags & SRE_FLAG_LOCALE:
+ raise ValueError("cannot use LOCALE flag with a str pattern")
+ if not flags & SRE_FLAG_ASCII:
+ flags |= SRE_FLAG_UNICODE
+ elif flags & SRE_FLAG_UNICODE:
+ raise ValueError("ASCII and UNICODE flags are incompatible")
+ else:
+ if flags & SRE_FLAG_UNICODE:
+ raise ValueError("cannot use UNICODE flag with a bytes pattern")
+ if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII:
+ raise ValueError("ASCII and LOCALE flags are incompatible")
+ return flags
+
+def parse(str, flags=0, state=None):
+ # parse 're' pattern into list of (opcode, argument) tuples
+
+ source = Tokenizer(str)
+
+ if state is None:
+ state = State()
+ state.flags = flags
+ state.str = str
+
+ p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
+ p.state.flags = fix_flags(str, p.state.flags)
+
+ if source.next is not None:
+ assert source.next == ")"
+ raise source.error("unbalanced parenthesis")
+
+ for g in p.state.grouprefpos:
+ if g >= p.state.groups:
+ msg = "invalid group reference %d" % g
+ raise error(msg, str, p.state.grouprefpos[g])
+
+ if flags & SRE_FLAG_DEBUG:
+ p.dump()
+
+ return p
+
+def parse_template(source, state):
+ # parse 're' replacement string into list of literals and
+ # group references
+ s = Tokenizer(source)
+ sget = s.get
+ groups = []
+ literals = []
+ literal = []
+ lappend = literal.append
+ def addgroup(index, pos):
+ if index > state.groups:
+ raise s.error("invalid group reference %d" % index, pos)
+ if literal:
+ literals.append(''.join(literal))
+ del literal[:]
+ groups.append((len(literals), index))
+ literals.append(None)
+ groupindex = state.groupindex
+ while True:
+ this = sget()
+ if this is None:
+ break # end of replacement string
+ if this[0] == "\\":
+ # group
+ c = this[1]
+ if c == "g":
+ if not s.match("<"):
+ raise s.error("missing <")
+ name = s.getuntil(">", "group name")
+ if name.isidentifier():
+ s.checkgroupname(name, 1, -1)
+ try:
+ index = groupindex[name]
+ except KeyError:
+ raise IndexError("unknown group name %r" % name) from None
+ else:
+ try:
+ index = int(name)
+ if index < 0:
+ raise ValueError
+ except ValueError:
+ raise s.error("bad character in group name %r" % name,
+ len(name) + 1) from None
+ if index >= MAXGROUPS:
+ raise s.error("invalid group reference %d" % index,
+ len(name) + 1)
+ if not (name.isdecimal() and name.isascii()):
+ import warnings
+ warnings.warn(
+ "bad character in group name %s at position %d" %
+ (repr(name) if s.istext else ascii(name),
+ s.tell() - len(name) - 1),
+ DeprecationWarning, stacklevel=5
+ )
+ addgroup(index, len(name) + 1)
+ elif c == "0":
+ if s.next in OCTDIGITS:
+ this += sget()
+ if s.next in OCTDIGITS:
+ this += sget()
+ lappend(chr(int(this[1:], 8) & 0xff))
+ elif c in DIGITS:
+ isoctal = False
+ if s.next in DIGITS:
+ this += sget()
+ if (c in OCTDIGITS and this[2] in OCTDIGITS and
+ s.next in OCTDIGITS):
+ this += sget()
+ isoctal = True
+ c = int(this[1:], 8)
+ if c > 0o377:
+ raise s.error('octal escape value %s outside of '
+ 'range 0-0o377' % this, len(this))
+ lappend(chr(c))
+ if not isoctal:
+ addgroup(int(this[1:]), len(this) - 1)
+ else:
+ try:
+ this = chr(ESCAPES[this][1])
+ except KeyError:
+ if c in ASCIILETTERS:
+ raise s.error('bad escape %s' % this, len(this)) from None
+ lappend(this)
+ else:
+ lappend(this)
+ if literal:
+ literals.append(''.join(literal))
+ if not isinstance(source, str):
+ # The tokenizer implicitly decodes bytes objects as latin-1, we must
+ # therefore re-encode the final representation.
+ literals = [None if s is None else s.encode('latin-1') for s in literals]
+ return groups, literals
+
+def expand_template(template, match):
+ g = match.group
+ empty = match.string[:0]
+ groups, literals = template
+ literals = literals[:]
+ try:
+ for index, group in groups:
+ literals[index] = g(group) or empty
+ except IndexError:
+ raise error("invalid group reference %d" % index) from None
+ return empty.join(literals)
diff --git a/common/py3-stdlib/reprlib.py b/common/py3-stdlib/reprlib.py
index 616b343..f3518df 100644
--- a/common/py3-stdlib/reprlib.py
+++ b/common/py3-stdlib/reprlib.py
@@ -36,6 +36,7 @@
class Repr:
def __init__(self):
+ self.fillvalue = '...'
self.maxlevel = 6
self.maxtuple = 6
self.maxlist = 6
@@ -64,14 +65,16 @@
def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
n = len(x)
if level <= 0 and n:
- s = '...'
+ s = self.fillvalue
else:
newlevel = level - 1
repr1 = self.repr1
pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
- if n > maxiter: pieces.append('...')
+ if n > maxiter:
+ pieces.append(self.fillvalue)
s = ', '.join(pieces)
- if n == 1 and trail: right = trail + right
+ if n == 1 and trail:
+ right = trail + right
return '%s%s%s' % (left, s, right)
def repr_tuple(self, x, level):
@@ -104,8 +107,10 @@
def repr_dict(self, x, level):
n = len(x)
- if n == 0: return '{}'
- if level <= 0: return '{...}'
+ if n == 0:
+ return '{}'
+ if level <= 0:
+ return '{' + self.fillvalue + '}'
newlevel = level - 1
repr1 = self.repr1
pieces = []
@@ -113,7 +118,8 @@
keyrepr = repr1(key, newlevel)
valrepr = repr1(x[key], newlevel)
pieces.append('%s: %s' % (keyrepr, valrepr))
- if n > self.maxdict: pieces.append('...')
+ if n > self.maxdict:
+ pieces.append(self.fillvalue)
s = ', '.join(pieces)
return '{%s}' % (s,)
@@ -123,7 +129,7 @@
i = max(0, (self.maxstring-3)//2)
j = max(0, self.maxstring-3-i)
s = builtins.repr(x[:i] + x[len(x)-j:])
- s = s[:i] + '...' + s[len(s)-j:]
+ s = s[:i] + self.fillvalue + s[len(s)-j:]
return s
def repr_int(self, x, level):
@@ -131,7 +137,7 @@
if len(s) > self.maxlong:
i = max(0, (self.maxlong-3)//2)
j = max(0, self.maxlong-3-i)
- s = s[:i] + '...' + s[len(s)-j:]
+ s = s[:i] + self.fillvalue + s[len(s)-j:]
return s
def repr_instance(self, x, level):
@@ -144,7 +150,7 @@
if len(s) > self.maxother:
i = max(0, (self.maxother-3)//2)
j = max(0, self.maxother-3-i)
- s = s[:i] + '...' + s[len(s)-j:]
+ s = s[:i] + self.fillvalue + s[len(s)-j:]
return s
diff --git a/common/py3-stdlib/rlcompleter.py b/common/py3-stdlib/rlcompleter.py
index 98b7930..206d6fb 100644
--- a/common/py3-stdlib/rlcompleter.py
+++ b/common/py3-stdlib/rlcompleter.py
@@ -32,6 +32,8 @@
import atexit
import builtins
import inspect
+import keyword
+import re
import __main__
__all__ = ["Completer"]
@@ -113,18 +115,17 @@
defined in self.namespace that match.
"""
- import keyword
matches = []
seen = {"__builtins__"}
n = len(text)
- for word in keyword.kwlist:
+ for word in keyword.kwlist + keyword.softkwlist:
if word[:n] == text:
seen.add(word)
if word in {'finally', 'try'}:
word = word + ':'
elif word not in {'False', 'None', 'True',
'break', 'continue', 'pass',
- 'else'}:
+ 'else', '_'}:
word = word + ' '
matches.append(word)
for nspace in [self.namespace, builtins.__dict__]:
@@ -146,7 +147,6 @@
with a __getattr__ hook is evaluated.
"""
- import re
m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text)
if not m:
return []
diff --git a/common/py3-stdlib/runpy.py b/common/py3-stdlib/runpy.py
index c7d3d8c..54fc136 100644
--- a/common/py3-stdlib/runpy.py
+++ b/common/py3-stdlib/runpy.py
@@ -14,18 +14,20 @@
import importlib.machinery # importlib first so we can test #15386 via -m
import importlib.util
import io
-import types
import os
__all__ = [
"run_module", "run_path",
]
+# avoid 'import types' just for ModuleType
+ModuleType = type(sys)
+
class _TempModule(object):
"""Temporarily replace a module in sys.modules with an empty namespace"""
def __init__(self, mod_name):
self.mod_name = mod_name
- self.module = types.ModuleType(mod_name)
+ self.module = ModuleType(mod_name)
self._saved_module = []
def __enter__(self):
diff --git a/common/py3-stdlib/secrets.py b/common/py3-stdlib/secrets.py
index a546efb..900381a 100644
--- a/common/py3-stdlib/secrets.py
+++ b/common/py3-stdlib/secrets.py
@@ -2,7 +2,7 @@
managing secrets such as account authentication, tokens, and similar.
See PEP 506 for more information.
-https://www.python.org/dev/peps/pep-0506/
+https://peps.python.org/pep-0506/
"""
diff --git a/common/py3-stdlib/selectors.py b/common/py3-stdlib/selectors.py
index bb15a1c..af6a4f9 100644
--- a/common/py3-stdlib/selectors.py
+++ b/common/py3-stdlib/selectors.py
@@ -50,12 +50,11 @@
Object used to associate a file object to its backing
file descriptor, selected event mask, and attached data.
"""
-if sys.version_info >= (3, 5):
- SelectorKey.fileobj.__doc__ = 'File object registered.'
- SelectorKey.fd.__doc__ = 'Underlying file descriptor.'
- SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.'
- SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object.
- For example, this could be used to store a per-client session ID.''')
+SelectorKey.fileobj.__doc__ = 'File object registered.'
+SelectorKey.fd.__doc__ = 'Underlying file descriptor.'
+SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.'
+SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object.
+For example, this could be used to store a per-client session ID.''')
class _SelectorMapping(Mapping):
diff --git a/common/py3-stdlib/shutil.py b/common/py3-stdlib/shutil.py
index b7bffa3..f4efaef 100644
--- a/common/py3-stdlib/shutil.py
+++ b/common/py3-stdlib/shutil.py
@@ -40,6 +40,8 @@
import nt
COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024
+# This should never be removed, see rationale in:
+# https://bugs.python.org/issue43743#msg393429
_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux")
_HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS
@@ -186,9 +188,9 @@
def copyfileobj(fsrc, fdst, length=0):
"""copy data from file-like object fsrc to file-like object fdst"""
- # Localize variable access to minimize overhead.
if not length:
length = COPY_BUFSIZE
+ # Localize variable access to minimize overhead.
fsrc_read = fsrc.read
fdst_write = fdst.write
while True:
@@ -685,9 +687,14 @@
os.scandir in os.supports_fd and
os.stat in os.supports_follow_symlinks)
-def rmtree(path, ignore_errors=False, onerror=None):
+def rmtree(path, ignore_errors=False, onerror=None, *, dir_fd=None):
"""Recursively delete a directory tree.
+ If dir_fd is not None, it should be a file descriptor open to a directory;
+ path will then be relative to that directory.
+ dir_fd may not be implemented on your platform.
+ If it is unavailable, using it will raise a NotImplementedError.
+
If ignore_errors is set, errors are ignored; otherwise, if onerror
is set, it is called to handle the error with arguments (func,
path, exc_info) where func is platform and implementation dependent;
@@ -696,7 +703,7 @@
is false and onerror is None, an exception is raised.
"""
- sys.audit("shutil.rmtree", path)
+ sys.audit("shutil.rmtree", path, dir_fd)
if ignore_errors:
def onerror(*args):
pass
@@ -710,12 +717,12 @@
# Note: To guard against symlink races, we use the standard
# lstat()/open()/fstat() trick.
try:
- orig_st = os.lstat(path)
+ orig_st = os.lstat(path, dir_fd=dir_fd)
except Exception:
onerror(os.lstat, path, sys.exc_info())
return
try:
- fd = os.open(path, os.O_RDONLY)
+ fd = os.open(path, os.O_RDONLY, dir_fd=dir_fd)
fd_closed = False
except Exception:
onerror(os.open, path, sys.exc_info())
@@ -726,7 +733,7 @@
try:
os.close(fd)
fd_closed = True
- os.rmdir(path)
+ os.rmdir(path, dir_fd=dir_fd)
except OSError:
onerror(os.rmdir, path, sys.exc_info())
else:
@@ -739,6 +746,8 @@
if not fd_closed:
os.close(fd)
else:
+ if dir_fd is not None:
+ raise NotImplementedError("dir_fd unavailable on this platform")
try:
if _rmtree_islink(path):
# symlinks to directories are forbidden, see bug #1669
@@ -1222,7 +1231,7 @@
finally:
zip.close()
-def _unpack_tarfile(filename, extract_dir):
+def _unpack_tarfile(filename, extract_dir, *, filter=None):
"""Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`
"""
import tarfile # late import for breaking circular dependency
@@ -1232,7 +1241,7 @@
raise ReadError(
"%s is not a compressed or uncompressed tar file" % filename)
try:
- tarobj.extractall(extract_dir)
+ tarobj.extractall(extract_dir, filter=filter)
finally:
tarobj.close()
@@ -1265,7 +1274,7 @@
return name
return None
-def unpack_archive(filename, extract_dir=None, format=None):
+def unpack_archive(filename, extract_dir=None, format=None, *, filter=None):
"""Unpack an archive.
`filename` is the name of the archive.
@@ -1279,6 +1288,9 @@
was registered for that extension.
In case none is found, a ValueError is raised.
+
+ If `filter` is given, it is passed to the underlying
+ extraction function.
"""
sys.audit("shutil.unpack_archive", filename, extract_dir, format)
@@ -1288,6 +1300,10 @@
extract_dir = os.fspath(extract_dir)
filename = os.fspath(filename)
+ if filter is None:
+ filter_kwargs = {}
+ else:
+ filter_kwargs = {'filter': filter}
if format is not None:
try:
format_info = _UNPACK_FORMATS[format]
@@ -1295,7 +1311,7 @@
raise ValueError("Unknown unpack format '{0}'".format(format)) from None
func = format_info[1]
- func(filename, extract_dir, **dict(format_info[2]))
+ func(filename, extract_dir, **dict(format_info[2]), **filter_kwargs)
else:
# we need to look at the registered unpackers supported extensions
format = _find_unpack_format(filename)
@@ -1303,7 +1319,7 @@
raise ReadError("Unknown archive format '{0}'".format(filename))
func = _UNPACK_FORMATS[format][1]
- kwargs = dict(_UNPACK_FORMATS[format][2])
+ kwargs = dict(_UNPACK_FORMATS[format][2]) | filter_kwargs
func(filename, extract_dir, **kwargs)
@@ -1414,9 +1430,9 @@
# os.get_terminal_size() is unsupported
size = os.terminal_size(fallback)
if columns <= 0:
- columns = size.columns
+ columns = size.columns or fallback[0]
if lines <= 0:
- lines = size.lines
+ lines = size.lines or fallback[1]
return os.terminal_size((columns, lines))
diff --git a/common/py3-stdlib/site.py b/common/py3-stdlib/site.py
index 939893e..69670d9 100644
--- a/common/py3-stdlib/site.py
+++ b/common/py3-stdlib/site.py
@@ -266,8 +266,8 @@
if env_base:
return env_base
- # VxWorks has no home directories
- if sys.platform == "vxworks":
+ # Emscripten, VxWorks, and WASI have no home directories
+ if sys.platform in {"emscripten", "vxworks", "wasi"}:
return None
def joinuser(*args):
@@ -361,11 +361,11 @@
continue
seen.add(prefix)
- libdirs = [sys.platlibdir]
- if sys.platlibdir != "lib":
- libdirs.append("lib")
-
if os.sep == '/':
+ libdirs = [sys.platlibdir]
+ if sys.platlibdir != "lib":
+ libdirs.append("lib")
+
for libdir in libdirs:
path = os.path.join(prefix, libdir,
"python%d.%d" % sys.version_info[:2],
@@ -373,10 +373,7 @@
sitepackages.append(path)
else:
sitepackages.append(prefix)
-
- for libdir in libdirs:
- path = os.path.join(prefix, libdir, "site-packages")
- sitepackages.append(path)
+ sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
return sitepackages
def addsitepackages(known_paths, prefixes=None):
@@ -418,8 +415,10 @@
files, dirs = [], []
# Not all modules are required to have a __file__ attribute. See
# PEP 420 for more details.
- if hasattr(os, '__file__'):
+ here = getattr(sys, '_stdlib_dir', None)
+ if not here and hasattr(os, '__file__'):
here = os.path.dirname(os.__file__)
+ if here:
files.extend(["LICENSE.txt", "LICENSE"])
dirs.extend([os.path.join(here, os.pardir), here, os.curdir])
builtins.license = _sitebuiltins._Printer(
diff --git a/common/py3-stdlib/smtpd.py b/common/py3-stdlib/smtpd.py
index 963e0a7..b23579f 100755
--- a/common/py3-stdlib/smtpd.py
+++ b/common/py3-stdlib/smtpd.py
@@ -60,13 +60,6 @@
# SMTP errors from the backend server at all. This should be fixed
# (contributions are welcome!).
#
-# MailmanProxy - An experimental hack to work with GNU Mailman
-# <www.list.org>. Using this server as your real incoming smtpd, your
-# mailhost will automatically recognize and accept mail destined to Mailman
-# lists when those lists are created. Every message not destined for a list
-# gets forwarded to a real backend smtpd, as with PureProxy. Again, errors
-# are not handled correctly yet.
-#
#
# Author: Barry Warsaw <barry@python.org>
#
@@ -84,20 +77,18 @@
import time
import socket
import collections
-from warnings import warn
+from warnings import _deprecated, warn
from email._header_value_parser import get_addr_spec, get_angle_addr
__all__ = [
"SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy",
- "MailmanProxy",
]
-warn(
- 'The smtpd module is deprecated and unmaintained and will be removed '
- 'in Python 3.12. Please see aiosmtpd '
- '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.',
- DeprecationWarning,
- stacklevel=2)
+_DEPRECATION_MSG = ('The {name} module is deprecated and unmaintained and will '
+ 'be removed in Python {remove}. Please see aiosmtpd '
+ '(https://aiosmtpd.readthedocs.io/) for the recommended '
+ 'replacement.')
+_deprecated(__name__, _DEPRECATION_MSG, remove=(3, 12))
# These are imported after the above warning so that users get the correct
@@ -789,91 +780,6 @@
return refused
-class MailmanProxy(PureProxy):
- def __init__(self, *args, **kwargs):
- warn('MailmanProxy is deprecated and will be removed '
- 'in future', DeprecationWarning, 2)
- if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']:
- raise ValueError("MailmanProxy does not support SMTPUTF8.")
- super(PureProxy, self).__init__(*args, **kwargs)
-
- def process_message(self, peer, mailfrom, rcpttos, data):
- from io import StringIO
- from Mailman import Utils
- from Mailman import Message
- from Mailman import MailList
- # If the message is to a Mailman mailing list, then we'll invoke the
- # Mailman script directly, without going through the real smtpd.
- # Otherwise we'll forward it to the local proxy for disposition.
- listnames = []
- for rcpt in rcpttos:
- local = rcpt.lower().split('@')[0]
- # We allow the following variations on the theme
- # listname
- # listname-admin
- # listname-owner
- # listname-request
- # listname-join
- # listname-leave
- parts = local.split('-')
- if len(parts) > 2:
- continue
- listname = parts[0]
- if len(parts) == 2:
- command = parts[1]
- else:
- command = ''
- if not Utils.list_exists(listname) or command not in (
- '', 'admin', 'owner', 'request', 'join', 'leave'):
- continue
- listnames.append((rcpt, listname, command))
- # Remove all list recipients from rcpttos and forward what we're not
- # going to take care of ourselves. Linear removal should be fine
- # since we don't expect a large number of recipients.
- for rcpt, listname, command in listnames:
- rcpttos.remove(rcpt)
- # If there's any non-list destined recipients left,
- print('forwarding recips:', ' '.join(rcpttos), file=DEBUGSTREAM)
- if rcpttos:
- refused = self._deliver(mailfrom, rcpttos, data)
- # TBD: what to do with refused addresses?
- print('we got refusals:', refused, file=DEBUGSTREAM)
- # Now deliver directly to the list commands
- mlists = {}
- s = StringIO(data)
- msg = Message.Message(s)
- # These headers are required for the proper execution of Mailman. All
- # MTAs in existence seem to add these if the original message doesn't
- # have them.
- if not msg.get('from'):
- msg['From'] = mailfrom
- if not msg.get('date'):
- msg['Date'] = time.ctime(time.time())
- for rcpt, listname, command in listnames:
- print('sending message to', rcpt, file=DEBUGSTREAM)
- mlist = mlists.get(listname)
- if not mlist:
- mlist = MailList.MailList(listname, lock=0)
- mlists[listname] = mlist
- # dispatch on the type of command
- if command == '':
- # post
- msg.Enqueue(mlist, tolist=1)
- elif command == 'admin':
- msg.Enqueue(mlist, toadmin=1)
- elif command == 'owner':
- msg.Enqueue(mlist, toowner=1)
- elif command == 'request':
- msg.Enqueue(mlist, torequest=1)
- elif command in ('join', 'leave'):
- # TBD: this is a hack!
- if command == 'join':
- msg['Subject'] = 'subscribe'
- else:
- msg['Subject'] = 'unsubscribe'
- msg.Enqueue(mlist, torequest=1)
-
-
class Options:
setuid = True
classname = 'PureProxy'
diff --git a/common/py3-stdlib/sndhdr.py b/common/py3-stdlib/sndhdr.py
index 96595c6..45def9a 100644
--- a/common/py3-stdlib/sndhdr.py
+++ b/common/py3-stdlib/sndhdr.py
@@ -27,6 +27,10 @@
explicitly given directories.
"""
+import warnings
+
+warnings._deprecated(__name__, remove=(3, 13))
+
# The file structure is top-down except that the test program and its
# subroutine come last.
@@ -73,7 +77,10 @@
tests = []
def test_aifc(h, f):
- import aifc
+ """AIFC and AIFF files"""
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import aifc
if not h.startswith(b'FORM'):
return None
if h[8:12] == b'AIFC':
@@ -94,6 +101,7 @@
def test_au(h, f):
+ """AU and SND files"""
if h.startswith(b'.snd'):
func = get_long_be
elif h[:4] in (b'\0ds.', b'dns.'):
@@ -127,6 +135,7 @@
def test_hcom(h, f):
+ """HCOM file"""
if h[65:69] != b'FSSD' or h[128:132] != b'HCOM':
return None
divisor = get_long_be(h[144:148])
@@ -140,6 +149,7 @@
def test_voc(h, f):
+ """VOC file"""
if not h.startswith(b'Creative Voice File\032'):
return None
sbseek = get_short_le(h[20:22])
@@ -154,6 +164,7 @@
def test_wav(h, f):
+ """WAV file"""
import wave
# 'RIFF' <len> 'WAVE' 'fmt ' <len>
if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ':
@@ -170,6 +181,7 @@
def test_8svx(h, f):
+ """8SVX file"""
if not h.startswith(b'FORM') or h[8:12] != b'8SVX':
return None
# Should decode it to get #channels -- assume always 1
@@ -179,6 +191,7 @@
def test_sndt(h, f):
+ """SNDT file"""
if h.startswith(b'SOUND'):
nsamples = get_long_le(h[8:12])
rate = get_short_le(h[20:22])
@@ -188,6 +201,7 @@
def test_sndr(h, f):
+ """SNDR file"""
if h.startswith(b'\0\0'):
rate = get_short_le(h[2:4])
if 4000 <= rate <= 25000:
diff --git a/common/py3-stdlib/socket.py b/common/py3-stdlib/socket.py
index 3796629..b5d46eb 100644
--- a/common/py3-stdlib/socket.py
+++ b/common/py3-stdlib/socket.py
@@ -13,7 +13,7 @@
socketpair() -- create a pair of new socket objects [*]
fromfd() -- create a socket object from an open file descriptor [*]
send_fds() -- Send file descriptor to the socket.
-recv_fds() -- Recieve file descriptors from the socket.
+recv_fds() -- Receive file descriptors from the socket.
fromshare() -- create a socket object from data received from socket.share() [*]
gethostname() -- return the current hostname
gethostbyname() -- map a hostname to its IP number
@@ -122,7 +122,7 @@
errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT
errorTab[10022] = "An invalid operation was attempted."
errorTab[10024] = "Too many open files."
- errorTab[10035] = "The socket operation would block"
+ errorTab[10035] = "The socket operation would block."
errorTab[10036] = "A blocking operation is already in progress."
errorTab[10037] = "Operation already in progress."
errorTab[10038] = "Socket operation on nonsocket."
@@ -254,17 +254,18 @@
self.type,
self.proto)
if not closed:
+ # getsockname and getpeername may not be available on WASI.
try:
laddr = self.getsockname()
if laddr:
s += ", laddr=%s" % str(laddr)
- except error:
+ except (error, AttributeError):
pass
try:
raddr = self.getpeername()
if raddr:
s += ", raddr=%s" % str(raddr)
- except error:
+ except (error, AttributeError):
pass
s += '>'
return s
@@ -806,7 +807,7 @@
_GLOBAL_DEFAULT_TIMEOUT = object()
def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
- source_address=None):
+ source_address=None, *, all_errors=False):
"""Connect to *address* and return the socket object.
Convenience function. Connect to *address* (a 2-tuple ``(host,
@@ -816,11 +817,13 @@
global default timeout setting returned by :func:`getdefaulttimeout`
is used. If *source_address* is set it must be a tuple of (host, port)
for the socket to bind as a source address before making the connection.
- A host of '' or port 0 tells the OS to use the default.
+ A host of '' or port 0 tells the OS to use the default. When a connection
+ cannot be created, raises the last error if *all_errors* is False,
+ and an ExceptionGroup of all errors if *all_errors* is True.
"""
host, port = address
- err = None
+ exceptions = []
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
af, socktype, proto, canonname, sa = res
sock = None
@@ -832,20 +835,24 @@
sock.bind(source_address)
sock.connect(sa)
# Break explicitly a reference cycle
- err = None
+ exceptions.clear()
return sock
- except error as _:
- err = _
+ except error as exc:
+ if not all_errors:
+ exceptions.clear() # raise only the last error
+ exceptions.append(exc)
if sock is not None:
sock.close()
- if err is not None:
+ if len(exceptions):
try:
- raise err
+ if not all_errors:
+ raise exceptions[0]
+ raise ExceptionGroup("create_connection failed", exceptions)
finally:
# Break explicitly a reference cycle
- err = None
+ exceptions.clear()
else:
raise error("getaddrinfo returns an empty list")
diff --git a/common/py3-stdlib/socketserver.py b/common/py3-stdlib/socketserver.py
index 0d9583d..30a5cfa 100644
--- a/common/py3-stdlib/socketserver.py
+++ b/common/py3-stdlib/socketserver.py
@@ -187,6 +187,7 @@
- address_family
- socket_type
- allow_reuse_address
+ - allow_reuse_port
Instance variables:
@@ -425,6 +426,7 @@
- socket_type
- request_queue_size (only for stream sockets)
- allow_reuse_address
+ - allow_reuse_port
Instance variables:
@@ -442,6 +444,8 @@
allow_reuse_address = False
+ allow_reuse_port = False
+
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
"""Constructor. May be extended, do not override."""
BaseServer.__init__(self, server_address, RequestHandlerClass)
@@ -461,8 +465,10 @@
May be overridden.
"""
- if self.allow_reuse_address:
+ if self.allow_reuse_address and hasattr(socket, "SO_REUSEADDR"):
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ if self.allow_reuse_port and hasattr(socket, "SO_REUSEPORT"):
+ self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
self.socket.bind(self.server_address)
self.server_address = self.socket.getsockname()
@@ -519,6 +525,8 @@
allow_reuse_address = False
+ allow_reuse_port = False
+
socket_type = socket.SOCK_DGRAM
max_packet_size = 8192
diff --git a/common/py3-stdlib/sqlite3/dbapi2.py b/common/py3-stdlib/sqlite3/dbapi2.py
index cfe6225..7cf4dd3 100644
--- a/common/py3-stdlib/sqlite3/dbapi2.py
+++ b/common/py3-stdlib/sqlite3/dbapi2.py
@@ -28,8 +28,6 @@
paramstyle = "qmark"
-threadsafety = 1
-
apilevel = "2.0"
Date = datetime.date
diff --git a/common/py3-stdlib/sqlite3/test/__init__.py b/common/py3-stdlib/sqlite3/test/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/common/py3-stdlib/sqlite3/test/__init__.py
+++ /dev/null
diff --git a/common/py3-stdlib/sqlite3/test/backup.py b/common/py3-stdlib/sqlite3/test/backup.py
deleted file mode 100644
index 4e30594..0000000
--- a/common/py3-stdlib/sqlite3/test/backup.py
+++ /dev/null
@@ -1,167 +0,0 @@
-import sqlite3 as sqlite
-import unittest
-
-
-class BackupTests(unittest.TestCase):
- def setUp(self):
- cx = self.cx = sqlite.connect(":memory:")
- cx.execute('CREATE TABLE foo (key INTEGER)')
- cx.executemany('INSERT INTO foo (key) VALUES (?)', [(3,), (4,)])
- cx.commit()
-
- def tearDown(self):
- self.cx.close()
-
- def verify_backup(self, bckcx):
- result = bckcx.execute("SELECT key FROM foo ORDER BY key").fetchall()
- self.assertEqual(result[0][0], 3)
- self.assertEqual(result[1][0], 4)
-
- def test_bad_target(self):
- with self.assertRaises(TypeError):
- self.cx.backup(None)
- with self.assertRaises(TypeError):
- self.cx.backup()
-
- def test_bad_target_filename(self):
- with self.assertRaises(TypeError):
- self.cx.backup('some_file_name.db')
-
- def test_bad_target_same_connection(self):
- with self.assertRaises(ValueError):
- self.cx.backup(self.cx)
-
- def test_bad_target_closed_connection(self):
- bck = sqlite.connect(':memory:')
- bck.close()
- with self.assertRaises(sqlite.ProgrammingError):
- self.cx.backup(bck)
-
- def test_bad_source_closed_connection(self):
- bck = sqlite.connect(':memory:')
- source = sqlite.connect(":memory:")
- source.close()
- with self.assertRaises(sqlite.ProgrammingError):
- source.backup(bck)
-
- def test_bad_target_in_transaction(self):
- bck = sqlite.connect(':memory:')
- bck.execute('CREATE TABLE bar (key INTEGER)')
- bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)])
- with self.assertRaises(sqlite.OperationalError) as cm:
- self.cx.backup(bck)
- if sqlite.sqlite_version_info < (3, 8, 8):
- self.assertEqual(str(cm.exception), 'target is in transaction')
-
- def test_keyword_only_args(self):
- with self.assertRaises(TypeError):
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, 1)
-
- def test_simple(self):
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck)
- self.verify_backup(bck)
-
- def test_progress(self):
- journal = []
-
- def progress(status, remaining, total):
- journal.append(status)
-
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, pages=1, progress=progress)
- self.verify_backup(bck)
-
- self.assertEqual(len(journal), 2)
- self.assertEqual(journal[0], sqlite.SQLITE_OK)
- self.assertEqual(journal[1], sqlite.SQLITE_DONE)
-
- def test_progress_all_pages_at_once_1(self):
- journal = []
-
- def progress(status, remaining, total):
- journal.append(remaining)
-
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, progress=progress)
- self.verify_backup(bck)
-
- self.assertEqual(len(journal), 1)
- self.assertEqual(journal[0], 0)
-
- def test_progress_all_pages_at_once_2(self):
- journal = []
-
- def progress(status, remaining, total):
- journal.append(remaining)
-
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, pages=-1, progress=progress)
- self.verify_backup(bck)
-
- self.assertEqual(len(journal), 1)
- self.assertEqual(journal[0], 0)
-
- def test_non_callable_progress(self):
- with self.assertRaises(TypeError) as cm:
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, pages=1, progress='bar')
- self.assertEqual(str(cm.exception), 'progress argument must be a callable')
-
- def test_modifying_progress(self):
- journal = []
-
- def progress(status, remaining, total):
- if not journal:
- self.cx.execute('INSERT INTO foo (key) VALUES (?)', (remaining+1000,))
- self.cx.commit()
- journal.append(remaining)
-
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, pages=1, progress=progress)
- self.verify_backup(bck)
-
- result = bck.execute("SELECT key FROM foo"
- " WHERE key >= 1000"
- " ORDER BY key").fetchall()
- self.assertEqual(result[0][0], 1001)
-
- self.assertEqual(len(journal), 3)
- self.assertEqual(journal[0], 1)
- self.assertEqual(journal[1], 1)
- self.assertEqual(journal[2], 0)
-
- def test_failing_progress(self):
- def progress(status, remaining, total):
- raise SystemError('nearly out of space')
-
- with self.assertRaises(SystemError) as err:
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, progress=progress)
- self.assertEqual(str(err.exception), 'nearly out of space')
-
- def test_database_source_name(self):
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, name='main')
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, name='temp')
- with self.assertRaises(sqlite.OperationalError) as cm:
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, name='non-existing')
- self.assertIn("unknown database", str(cm.exception))
-
- self.cx.execute("ATTACH DATABASE ':memory:' AS attached_db")
- self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)')
- self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)])
- self.cx.commit()
- with sqlite.connect(':memory:') as bck:
- self.cx.backup(bck, name='attached_db')
- self.verify_backup(bck)
-
-
-def suite():
- return unittest.TestLoader().loadTestsFromTestCase(BackupTests)
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/common/py3-stdlib/sqlite3/test/dbapi.py b/common/py3-stdlib/sqlite3/test/dbapi.py
deleted file mode 100644
index a764c82..0000000
--- a/common/py3-stdlib/sqlite3/test/dbapi.py
+++ /dev/null
@@ -1,1067 +0,0 @@
-# pysqlite2/test/dbapi.py: tests for DB-API compliance
-#
-# Copyright (C) 2004-2010 Gerhard Häring <gh@ghaering.de>
-#
-# This file is part of pysqlite.
-#
-# This software is provided 'as-is', without any express or implied
-# warranty. In no event will the authors be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-# claim that you wrote the original software. If you use this software
-# in a product, an acknowledgment in the product documentation would be
-# appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-# misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-
-import subprocess
-import threading
-import unittest
-import sqlite3 as sqlite
-import sys
-
-from test.support import check_disallow_instantiation, SHORT_TIMEOUT
-from test.support.os_helper import TESTFN, unlink
-
-
-class ModuleTests(unittest.TestCase):
- def test_api_level(self):
- self.assertEqual(sqlite.apilevel, "2.0",
- "apilevel is %s, should be 2.0" % sqlite.apilevel)
-
- def test_thread_safety(self):
- self.assertEqual(sqlite.threadsafety, 1,
- "threadsafety is %d, should be 1" % sqlite.threadsafety)
-
- def test_param_style(self):
- self.assertEqual(sqlite.paramstyle, "qmark",
- "paramstyle is '%s', should be 'qmark'" %
- sqlite.paramstyle)
-
- def test_warning(self):
- self.assertTrue(issubclass(sqlite.Warning, Exception),
- "Warning is not a subclass of Exception")
-
- def test_error(self):
- self.assertTrue(issubclass(sqlite.Error, Exception),
- "Error is not a subclass of Exception")
-
- def test_interface_error(self):
- self.assertTrue(issubclass(sqlite.InterfaceError, sqlite.Error),
- "InterfaceError is not a subclass of Error")
-
- def test_database_error(self):
- self.assertTrue(issubclass(sqlite.DatabaseError, sqlite.Error),
- "DatabaseError is not a subclass of Error")
-
- def test_data_error(self):
- self.assertTrue(issubclass(sqlite.DataError, sqlite.DatabaseError),
- "DataError is not a subclass of DatabaseError")
-
- def test_operational_error(self):
- self.assertTrue(issubclass(sqlite.OperationalError, sqlite.DatabaseError),
- "OperationalError is not a subclass of DatabaseError")
-
- def test_integrity_error(self):
- self.assertTrue(issubclass(sqlite.IntegrityError, sqlite.DatabaseError),
- "IntegrityError is not a subclass of DatabaseError")
-
- def test_internal_error(self):
- self.assertTrue(issubclass(sqlite.InternalError, sqlite.DatabaseError),
- "InternalError is not a subclass of DatabaseError")
-
- def test_programming_error(self):
- self.assertTrue(issubclass(sqlite.ProgrammingError, sqlite.DatabaseError),
- "ProgrammingError is not a subclass of DatabaseError")
-
- def test_not_supported_error(self):
- self.assertTrue(issubclass(sqlite.NotSupportedError,
- sqlite.DatabaseError),
- "NotSupportedError is not a subclass of DatabaseError")
-
- # sqlite3_enable_shared_cache() is deprecated on macOS and calling it may raise
- # OperationalError on some buildbots.
- @unittest.skipIf(sys.platform == "darwin", "shared cache is deprecated on macOS")
- def test_shared_cache_deprecated(self):
- for enable in (True, False):
- with self.assertWarns(DeprecationWarning) as cm:
- sqlite.enable_shared_cache(enable)
- self.assertIn("dbapi.py", cm.filename)
-
- def test_disallow_instantiation(self):
- cx = sqlite.connect(":memory:")
- check_disallow_instantiation(self, type(cx("select 1")))
-
-
-class ConnectionTests(unittest.TestCase):
-
- def setUp(self):
- self.cx = sqlite.connect(":memory:")
- cu = self.cx.cursor()
- cu.execute("create table test(id integer primary key, name text)")
- cu.execute("insert into test(name) values (?)", ("foo",))
-
- def tearDown(self):
- self.cx.close()
-
- def test_commit(self):
- self.cx.commit()
-
- def test_commit_after_no_changes(self):
- """
- A commit should also work when no changes were made to the database.
- """
- self.cx.commit()
- self.cx.commit()
-
- def test_rollback(self):
- self.cx.rollback()
-
- def test_rollback_after_no_changes(self):
- """
- A rollback should also work when no changes were made to the database.
- """
- self.cx.rollback()
- self.cx.rollback()
-
- def test_cursor(self):
- cu = self.cx.cursor()
-
- def test_failed_open(self):
- YOU_CANNOT_OPEN_THIS = "/foo/bar/bla/23534/mydb.db"
- with self.assertRaises(sqlite.OperationalError):
- con = sqlite.connect(YOU_CANNOT_OPEN_THIS)
-
- def test_close(self):
- self.cx.close()
-
- def test_exceptions(self):
- # Optional DB-API extension.
- self.assertEqual(self.cx.Warning, sqlite.Warning)
- self.assertEqual(self.cx.Error, sqlite.Error)
- self.assertEqual(self.cx.InterfaceError, sqlite.InterfaceError)
- self.assertEqual(self.cx.DatabaseError, sqlite.DatabaseError)
- self.assertEqual(self.cx.DataError, sqlite.DataError)
- self.assertEqual(self.cx.OperationalError, sqlite.OperationalError)
- self.assertEqual(self.cx.IntegrityError, sqlite.IntegrityError)
- self.assertEqual(self.cx.InternalError, sqlite.InternalError)
- self.assertEqual(self.cx.ProgrammingError, sqlite.ProgrammingError)
- self.assertEqual(self.cx.NotSupportedError, sqlite.NotSupportedError)
-
- def test_in_transaction(self):
- # Can't use db from setUp because we want to test initial state.
- cx = sqlite.connect(":memory:")
- cu = cx.cursor()
- self.assertEqual(cx.in_transaction, False)
- cu.execute("create table transactiontest(id integer primary key, name text)")
- self.assertEqual(cx.in_transaction, False)
- cu.execute("insert into transactiontest(name) values (?)", ("foo",))
- self.assertEqual(cx.in_transaction, True)
- cu.execute("select name from transactiontest where name=?", ["foo"])
- row = cu.fetchone()
- self.assertEqual(cx.in_transaction, True)
- cx.commit()
- self.assertEqual(cx.in_transaction, False)
- cu.execute("select name from transactiontest where name=?", ["foo"])
- row = cu.fetchone()
- self.assertEqual(cx.in_transaction, False)
-
- def test_in_transaction_ro(self):
- with self.assertRaises(AttributeError):
- self.cx.in_transaction = True
-
- def test_open_with_path_like_object(self):
- """ Checks that we can successfully connect to a database using an object that
- is PathLike, i.e. has __fspath__(). """
- self.addCleanup(unlink, TESTFN)
- class Path:
- def __fspath__(self):
- return TESTFN
- path = Path()
- with sqlite.connect(path) as cx:
- cx.execute('create table test(id integer)')
-
- def test_open_uri(self):
- self.addCleanup(unlink, TESTFN)
- with sqlite.connect(TESTFN) as cx:
- cx.execute('create table test(id integer)')
- with sqlite.connect('file:' + TESTFN, uri=True) as cx:
- cx.execute('insert into test(id) values(0)')
- with sqlite.connect('file:' + TESTFN + '?mode=ro', uri=True) as cx:
- with self.assertRaises(sqlite.OperationalError):
- cx.execute('insert into test(id) values(1)')
-
-
-class UninitialisedConnectionTests(unittest.TestCase):
- def setUp(self):
- self.cx = sqlite.Connection.__new__(sqlite.Connection)
-
- def test_uninit_operations(self):
- funcs = (
- lambda: self.cx.isolation_level,
- lambda: self.cx.total_changes,
- lambda: self.cx.in_transaction,
- lambda: self.cx.iterdump(),
- lambda: self.cx.cursor(),
- lambda: self.cx.close(),
- )
- for func in funcs:
- with self.subTest(func=func):
- self.assertRaisesRegex(sqlite.ProgrammingError,
- "Base Connection.__init__ not called",
- func)
-
-
-class CursorTests(unittest.TestCase):
- def setUp(self):
- self.cx = sqlite.connect(":memory:")
- self.cu = self.cx.cursor()
- self.cu.execute(
- "create table test(id integer primary key, name text, "
- "income number, unique_test text unique)"
- )
- self.cu.execute("insert into test(name) values (?)", ("foo",))
-
- def tearDown(self):
- self.cu.close()
- self.cx.close()
-
- def test_execute_no_args(self):
- self.cu.execute("delete from test")
-
- def test_execute_illegal_sql(self):
- with self.assertRaises(sqlite.OperationalError):
- self.cu.execute("select asdf")
-
- def test_execute_too_much_sql(self):
- with self.assertRaises(sqlite.Warning):
- self.cu.execute("select 5+4; select 4+5")
-
- def test_execute_too_much_sql2(self):
- self.cu.execute("select 5+4; -- foo bar")
-
- def test_execute_too_much_sql3(self):
- self.cu.execute("""
- select 5+4;
-
- /*
- foo
- */
- """)
-
- def test_execute_wrong_sql_arg(self):
- with self.assertRaises(TypeError):
- self.cu.execute(42)
-
- def test_execute_arg_int(self):
- self.cu.execute("insert into test(id) values (?)", (42,))
-
- def test_execute_arg_float(self):
- self.cu.execute("insert into test(income) values (?)", (2500.32,))
-
- def test_execute_arg_string(self):
- self.cu.execute("insert into test(name) values (?)", ("Hugo",))
-
- def test_execute_arg_string_with_zero_byte(self):
- self.cu.execute("insert into test(name) values (?)", ("Hu\x00go",))
-
- self.cu.execute("select name from test where id=?", (self.cu.lastrowid,))
- row = self.cu.fetchone()
- self.assertEqual(row[0], "Hu\x00go")
-
- def test_execute_non_iterable(self):
- with self.assertRaises(ValueError) as cm:
- self.cu.execute("insert into test(id) values (?)", 42)
- self.assertEqual(str(cm.exception), 'parameters are of unsupported type')
-
- def test_execute_wrong_no_of_args1(self):
- # too many parameters
- with self.assertRaises(sqlite.ProgrammingError):
- self.cu.execute("insert into test(id) values (?)", (17, "Egon"))
-
- def test_execute_wrong_no_of_args2(self):
- # too little parameters
- with self.assertRaises(sqlite.ProgrammingError):
- self.cu.execute("insert into test(id) values (?)")
-
- def test_execute_wrong_no_of_args3(self):
- # no parameters, parameters are needed
- with self.assertRaises(sqlite.ProgrammingError):
- self.cu.execute("insert into test(id) values (?)")
-
- def test_execute_param_list(self):
- self.cu.execute("insert into test(name) values ('foo')")
- self.cu.execute("select name from test where name=?", ["foo"])
- row = self.cu.fetchone()
- self.assertEqual(row[0], "foo")
-
- def test_execute_param_sequence(self):
- class L:
- def __len__(self):
- return 1
- def __getitem__(self, x):
- assert x == 0
- return "foo"
-
- self.cu.execute("insert into test(name) values ('foo')")
- self.cu.execute("select name from test where name=?", L())
- row = self.cu.fetchone()
- self.assertEqual(row[0], "foo")
-
- def test_execute_param_sequence_bad_len(self):
- # Issue41662: Error in __len__() was overridden with ProgrammingError.
- class L:
- def __len__(self):
- 1/0
- def __getitem__(slf, x):
- raise AssertionError
-
- self.cu.execute("insert into test(name) values ('foo')")
- with self.assertRaises(ZeroDivisionError):
- self.cu.execute("select name from test where name=?", L())
-
- def test_execute_dict_mapping(self):
- self.cu.execute("insert into test(name) values ('foo')")
- self.cu.execute("select name from test where name=:name", {"name": "foo"})
- row = self.cu.fetchone()
- self.assertEqual(row[0], "foo")
-
- def test_execute_dict_mapping_mapping(self):
- class D(dict):
- def __missing__(self, key):
- return "foo"
-
- self.cu.execute("insert into test(name) values ('foo')")
- self.cu.execute("select name from test where name=:name", D())
- row = self.cu.fetchone()
- self.assertEqual(row[0], "foo")
-
- def test_execute_dict_mapping_too_little_args(self):
- self.cu.execute("insert into test(name) values ('foo')")
- with self.assertRaises(sqlite.ProgrammingError):
- self.cu.execute("select name from test where name=:name and id=:id", {"name": "foo"})
-
- def test_execute_dict_mapping_no_args(self):
- self.cu.execute("insert into test(name) values ('foo')")
- with self.assertRaises(sqlite.ProgrammingError):
- self.cu.execute("select name from test where name=:name")
-
- def test_execute_dict_mapping_unnamed(self):
- self.cu.execute("insert into test(name) values ('foo')")
- with self.assertRaises(sqlite.ProgrammingError):
- self.cu.execute("select name from test where name=?", {"name": "foo"})
-
- def test_close(self):
- self.cu.close()
-
- def test_rowcount_execute(self):
- self.cu.execute("delete from test")
- self.cu.execute("insert into test(name) values ('foo')")
- self.cu.execute("insert into test(name) values ('foo')")
- self.cu.execute("update test set name='bar'")
- self.assertEqual(self.cu.rowcount, 2)
-
- def test_rowcount_select(self):
- """
- pysqlite does not know the rowcount of SELECT statements, because we
- don't fetch all rows after executing the select statement. The rowcount
- has thus to be -1.
- """
- self.cu.execute("select 5 union select 6")
- self.assertEqual(self.cu.rowcount, -1)
-
- def test_rowcount_executemany(self):
- self.cu.execute("delete from test")
- self.cu.executemany("insert into test(name) values (?)", [(1,), (2,), (3,)])
- self.assertEqual(self.cu.rowcount, 3)
-
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 35, 0),
- "Requires SQLite 3.35.0 or newer")
- def test_rowcount_update_returning(self):
- # gh-93421: rowcount is updated correctly for UPDATE...RETURNING queries
- self.cu.execute("update test set name='bar' where name='foo' returning 1")
- self.assertEqual(self.cu.fetchone()[0], 1)
- self.assertEqual(self.cu.rowcount, 1)
-
- def test_total_changes(self):
- self.cu.execute("insert into test(name) values ('foo')")
- self.cu.execute("insert into test(name) values ('foo')")
- self.assertLess(2, self.cx.total_changes, msg='total changes reported wrong value')
-
- # Checks for executemany:
- # Sequences are required by the DB-API, iterators
- # enhancements in pysqlite.
-
- def test_execute_many_sequence(self):
- self.cu.executemany("insert into test(income) values (?)", [(x,) for x in range(100, 110)])
-
- def test_execute_many_iterator(self):
- class MyIter:
- def __init__(self):
- self.value = 5
-
- def __iter__(self):
- return self
-
- def __next__(self):
- if self.value == 10:
- raise StopIteration
- else:
- self.value += 1
- return (self.value,)
-
- self.cu.executemany("insert into test(income) values (?)", MyIter())
-
- def test_execute_many_generator(self):
- def mygen():
- for i in range(5):
- yield (i,)
-
- self.cu.executemany("insert into test(income) values (?)", mygen())
-
- def test_execute_many_wrong_sql_arg(self):
- with self.assertRaises(TypeError):
- self.cu.executemany(42, [(3,)])
-
- def test_execute_many_select(self):
- with self.assertRaises(sqlite.ProgrammingError):
- self.cu.executemany("select ?", [(3,)])
-
- def test_execute_many_not_iterable(self):
- with self.assertRaises(TypeError):
- self.cu.executemany("insert into test(income) values (?)", 42)
-
- def test_fetch_iter(self):
- # Optional DB-API extension.
- self.cu.execute("delete from test")
- self.cu.execute("insert into test(id) values (?)", (5,))
- self.cu.execute("insert into test(id) values (?)", (6,))
- self.cu.execute("select id from test order by id")
- lst = []
- for row in self.cu:
- lst.append(row[0])
- self.assertEqual(lst[0], 5)
- self.assertEqual(lst[1], 6)
-
- def test_fetchone(self):
- self.cu.execute("select name from test")
- row = self.cu.fetchone()
- self.assertEqual(row[0], "foo")
- row = self.cu.fetchone()
- self.assertEqual(row, None)
-
- def test_fetchone_no_statement(self):
- cur = self.cx.cursor()
- row = cur.fetchone()
- self.assertEqual(row, None)
-
- def test_array_size(self):
- # must default to 1
- self.assertEqual(self.cu.arraysize, 1)
-
- # now set to 2
- self.cu.arraysize = 2
-
- # now make the query return 3 rows
- self.cu.execute("delete from test")
- self.cu.execute("insert into test(name) values ('A')")
- self.cu.execute("insert into test(name) values ('B')")
- self.cu.execute("insert into test(name) values ('C')")
- self.cu.execute("select name from test")
- res = self.cu.fetchmany()
-
- self.assertEqual(len(res), 2)
-
- def test_fetchmany(self):
- self.cu.execute("select name from test")
- res = self.cu.fetchmany(100)
- self.assertEqual(len(res), 1)
- res = self.cu.fetchmany(100)
- self.assertEqual(res, [])
-
- def test_fetchmany_kw_arg(self):
- """Checks if fetchmany works with keyword arguments"""
- self.cu.execute("select name from test")
- res = self.cu.fetchmany(size=100)
- self.assertEqual(len(res), 1)
-
- def test_fetchall(self):
- self.cu.execute("select name from test")
- res = self.cu.fetchall()
- self.assertEqual(len(res), 1)
- res = self.cu.fetchall()
- self.assertEqual(res, [])
-
- def test_setinputsizes(self):
- self.cu.setinputsizes([3, 4, 5])
-
- def test_setoutputsize(self):
- self.cu.setoutputsize(5, 0)
-
- def test_setoutputsize_no_column(self):
- self.cu.setoutputsize(42)
-
- def test_cursor_connection(self):
- # Optional DB-API extension.
- self.assertEqual(self.cu.connection, self.cx)
-
- def test_wrong_cursor_callable(self):
- with self.assertRaises(TypeError):
- def f(): pass
- cur = self.cx.cursor(f)
-
- def test_cursor_wrong_class(self):
- class Foo: pass
- foo = Foo()
- with self.assertRaises(TypeError):
- cur = sqlite.Cursor(foo)
-
- def test_last_row_id_on_replace(self):
- """
- INSERT OR REPLACE and REPLACE INTO should produce the same behavior.
- """
- sql = '{} INTO test(id, unique_test) VALUES (?, ?)'
- for statement in ('INSERT OR REPLACE', 'REPLACE'):
- with self.subTest(statement=statement):
- self.cu.execute(sql.format(statement), (1, 'foo'))
- self.assertEqual(self.cu.lastrowid, 1)
-
- def test_last_row_id_on_ignore(self):
- self.cu.execute(
- "insert or ignore into test(unique_test) values (?)",
- ('test',))
- self.assertEqual(self.cu.lastrowid, 2)
- self.cu.execute(
- "insert or ignore into test(unique_test) values (?)",
- ('test',))
- self.assertEqual(self.cu.lastrowid, 2)
-
- def test_last_row_id_insert_o_r(self):
- results = []
- for statement in ('FAIL', 'ABORT', 'ROLLBACK'):
- sql = 'INSERT OR {} INTO test(unique_test) VALUES (?)'
- with self.subTest(statement='INSERT OR {}'.format(statement)):
- self.cu.execute(sql.format(statement), (statement,))
- results.append((statement, self.cu.lastrowid))
- with self.assertRaises(sqlite.IntegrityError):
- self.cu.execute(sql.format(statement), (statement,))
- results.append((statement, self.cu.lastrowid))
- expected = [
- ('FAIL', 2), ('FAIL', 2),
- ('ABORT', 3), ('ABORT', 3),
- ('ROLLBACK', 4), ('ROLLBACK', 4),
- ]
- self.assertEqual(results, expected)
-
-
-class ThreadTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- self.cur = self.con.cursor()
- self.cur.execute("create table test(id integer primary key, name text, bin binary, ratio number, ts timestamp)")
-
- def tearDown(self):
- self.cur.close()
- self.con.close()
-
- def test_con_cursor(self):
- def run(con, errors):
- try:
- cur = con.cursor()
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- t = threading.Thread(target=run, kwargs={"con": self.con, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
- def test_con_commit(self):
- def run(con, errors):
- try:
- con.commit()
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- t = threading.Thread(target=run, kwargs={"con": self.con, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
- def test_con_rollback(self):
- def run(con, errors):
- try:
- con.rollback()
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- t = threading.Thread(target=run, kwargs={"con": self.con, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
- def test_con_close(self):
- def run(con, errors):
- try:
- con.close()
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- t = threading.Thread(target=run, kwargs={"con": self.con, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
- def test_cur_implicit_begin(self):
- def run(cur, errors):
- try:
- cur.execute("insert into test(name) values ('a')")
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- t = threading.Thread(target=run, kwargs={"cur": self.cur, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
- def test_cur_close(self):
- def run(cur, errors):
- try:
- cur.close()
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- t = threading.Thread(target=run, kwargs={"cur": self.cur, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
- def test_cur_execute(self):
- def run(cur, errors):
- try:
- cur.execute("select name from test")
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- self.cur.execute("insert into test(name) values ('a')")
- t = threading.Thread(target=run, kwargs={"cur": self.cur, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
- def test_cur_iter_next(self):
- def run(cur, errors):
- try:
- row = cur.fetchone()
- errors.append("did not raise ProgrammingError")
- return
- except sqlite.ProgrammingError:
- return
- except:
- errors.append("raised wrong exception")
-
- errors = []
- self.cur.execute("insert into test(name) values ('a')")
- self.cur.execute("select name from test")
- t = threading.Thread(target=run, kwargs={"cur": self.cur, "errors": errors})
- t.start()
- t.join()
- if len(errors) > 0:
- self.fail("\n".join(errors))
-
-class ConstructorTests(unittest.TestCase):
- def test_date(self):
- d = sqlite.Date(2004, 10, 28)
-
- def test_time(self):
- t = sqlite.Time(12, 39, 35)
-
- def test_timestamp(self):
- ts = sqlite.Timestamp(2004, 10, 28, 12, 39, 35)
-
- def test_date_from_ticks(self):
- d = sqlite.DateFromTicks(42)
-
- def test_time_from_ticks(self):
- t = sqlite.TimeFromTicks(42)
-
- def test_timestamp_from_ticks(self):
- ts = sqlite.TimestampFromTicks(42)
-
- def test_binary(self):
- b = sqlite.Binary(b"\0'")
-
-class ExtensionTests(unittest.TestCase):
- def test_script_string_sql(self):
- con = sqlite.connect(":memory:")
- cur = con.cursor()
- cur.executescript("""
- -- bla bla
- /* a stupid comment */
- create table a(i);
- insert into a(i) values (5);
- """)
- cur.execute("select i from a")
- res = cur.fetchone()[0]
- self.assertEqual(res, 5)
-
- def test_script_syntax_error(self):
- con = sqlite.connect(":memory:")
- cur = con.cursor()
- with self.assertRaises(sqlite.OperationalError):
- cur.executescript("create table test(x); asdf; create table test2(x)")
-
- def test_script_error_normal(self):
- con = sqlite.connect(":memory:")
- cur = con.cursor()
- with self.assertRaises(sqlite.OperationalError):
- cur.executescript("create table test(sadfsadfdsa); select foo from hurz;")
-
- def test_cursor_executescript_as_bytes(self):
- con = sqlite.connect(":memory:")
- cur = con.cursor()
- with self.assertRaises(ValueError) as cm:
- cur.executescript(b"create table test(foo); insert into test(foo) values (5);")
- self.assertEqual(str(cm.exception), 'script argument must be unicode.')
-
- def test_connection_execute(self):
- con = sqlite.connect(":memory:")
- result = con.execute("select 5").fetchone()[0]
- self.assertEqual(result, 5, "Basic test of Connection.execute")
-
- def test_connection_executemany(self):
- con = sqlite.connect(":memory:")
- con.execute("create table test(foo)")
- con.executemany("insert into test(foo) values (?)", [(3,), (4,)])
- result = con.execute("select foo from test order by foo").fetchall()
- self.assertEqual(result[0][0], 3, "Basic test of Connection.executemany")
- self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany")
-
- def test_connection_executescript(self):
- con = sqlite.connect(":memory:")
- con.executescript("create table test(foo); insert into test(foo) values (5);")
- result = con.execute("select foo from test").fetchone()[0]
- self.assertEqual(result, 5, "Basic test of Connection.executescript")
-
-class ClosedConTests(unittest.TestCase):
- def test_closed_con_cursor(self):
- con = sqlite.connect(":memory:")
- con.close()
- with self.assertRaises(sqlite.ProgrammingError):
- cur = con.cursor()
-
- def test_closed_con_commit(self):
- con = sqlite.connect(":memory:")
- con.close()
- with self.assertRaises(sqlite.ProgrammingError):
- con.commit()
-
- def test_closed_con_rollback(self):
- con = sqlite.connect(":memory:")
- con.close()
- with self.assertRaises(sqlite.ProgrammingError):
- con.rollback()
-
- def test_closed_cur_execute(self):
- con = sqlite.connect(":memory:")
- cur = con.cursor()
- con.close()
- with self.assertRaises(sqlite.ProgrammingError):
- cur.execute("select 4")
-
- def test_closed_create_function(self):
- con = sqlite.connect(":memory:")
- con.close()
- def f(x): return 17
- with self.assertRaises(sqlite.ProgrammingError):
- con.create_function("foo", 1, f)
-
- def test_closed_create_aggregate(self):
- con = sqlite.connect(":memory:")
- con.close()
- class Agg:
- def __init__(self):
- pass
- def step(self, x):
- pass
- def finalize(self):
- return 17
- with self.assertRaises(sqlite.ProgrammingError):
- con.create_aggregate("foo", 1, Agg)
-
- def test_closed_set_authorizer(self):
- con = sqlite.connect(":memory:")
- con.close()
- def authorizer(*args):
- return sqlite.DENY
- with self.assertRaises(sqlite.ProgrammingError):
- con.set_authorizer(authorizer)
-
- def test_closed_set_progress_callback(self):
- con = sqlite.connect(":memory:")
- con.close()
- def progress(): pass
- with self.assertRaises(sqlite.ProgrammingError):
- con.set_progress_handler(progress, 100)
-
- def test_closed_call(self):
- con = sqlite.connect(":memory:")
- con.close()
- with self.assertRaises(sqlite.ProgrammingError):
- con()
-
-class ClosedCurTests(unittest.TestCase):
- def test_closed(self):
- con = sqlite.connect(":memory:")
- cur = con.cursor()
- cur.close()
-
- for method_name in ("execute", "executemany", "executescript", "fetchall", "fetchmany", "fetchone"):
- if method_name in ("execute", "executescript"):
- params = ("select 4 union select 5",)
- elif method_name == "executemany":
- params = ("insert into foo(bar) values (?)", [(3,), (4,)])
- else:
- params = []
-
- with self.assertRaises(sqlite.ProgrammingError):
- method = getattr(cur, method_name)
- method(*params)
-
-
-class SqliteOnConflictTests(unittest.TestCase):
- """
- Tests for SQLite's "insert on conflict" feature.
-
- See https://www.sqlite.org/lang_conflict.html for details.
- """
-
- def setUp(self):
- self.cx = sqlite.connect(":memory:")
- self.cu = self.cx.cursor()
- self.cu.execute("""
- CREATE TABLE test(
- id INTEGER PRIMARY KEY, name TEXT, unique_name TEXT UNIQUE
- );
- """)
-
- def tearDown(self):
- self.cu.close()
- self.cx.close()
-
- def test_on_conflict_rollback_with_explicit_transaction(self):
- self.cx.isolation_level = None # autocommit mode
- self.cu = self.cx.cursor()
- # Start an explicit transaction.
- self.cu.execute("BEGIN")
- self.cu.execute("INSERT INTO test(name) VALUES ('abort_test')")
- self.cu.execute("INSERT OR ROLLBACK INTO test(unique_name) VALUES ('foo')")
- with self.assertRaises(sqlite.IntegrityError):
- self.cu.execute("INSERT OR ROLLBACK INTO test(unique_name) VALUES ('foo')")
- # Use connection to commit.
- self.cx.commit()
- self.cu.execute("SELECT name, unique_name from test")
- # Transaction should have rolled back and nothing should be in table.
- self.assertEqual(self.cu.fetchall(), [])
-
- def test_on_conflict_abort_raises_with_explicit_transactions(self):
- # Abort cancels the current sql statement but doesn't change anything
- # about the current transaction.
- self.cx.isolation_level = None # autocommit mode
- self.cu = self.cx.cursor()
- # Start an explicit transaction.
- self.cu.execute("BEGIN")
- self.cu.execute("INSERT INTO test(name) VALUES ('abort_test')")
- self.cu.execute("INSERT OR ABORT INTO test(unique_name) VALUES ('foo')")
- with self.assertRaises(sqlite.IntegrityError):
- self.cu.execute("INSERT OR ABORT INTO test(unique_name) VALUES ('foo')")
- self.cx.commit()
- self.cu.execute("SELECT name, unique_name FROM test")
- # Expect the first two inserts to work, third to do nothing.
- self.assertEqual(self.cu.fetchall(), [('abort_test', None), (None, 'foo',)])
-
- def test_on_conflict_rollback_without_transaction(self):
- # Start of implicit transaction
- self.cu.execute("INSERT INTO test(name) VALUES ('abort_test')")
- self.cu.execute("INSERT OR ROLLBACK INTO test(unique_name) VALUES ('foo')")
- with self.assertRaises(sqlite.IntegrityError):
- self.cu.execute("INSERT OR ROLLBACK INTO test(unique_name) VALUES ('foo')")
- self.cu.execute("SELECT name, unique_name FROM test")
- # Implicit transaction is rolled back on error.
- self.assertEqual(self.cu.fetchall(), [])
-
- def test_on_conflict_abort_raises_without_transactions(self):
- # Abort cancels the current sql statement but doesn't change anything
- # about the current transaction.
- self.cu.execute("INSERT INTO test(name) VALUES ('abort_test')")
- self.cu.execute("INSERT OR ABORT INTO test(unique_name) VALUES ('foo')")
- with self.assertRaises(sqlite.IntegrityError):
- self.cu.execute("INSERT OR ABORT INTO test(unique_name) VALUES ('foo')")
- # Make sure all other values were inserted.
- self.cu.execute("SELECT name, unique_name FROM test")
- self.assertEqual(self.cu.fetchall(), [('abort_test', None), (None, 'foo',)])
-
- def test_on_conflict_fail(self):
- self.cu.execute("INSERT OR FAIL INTO test(unique_name) VALUES ('foo')")
- with self.assertRaises(sqlite.IntegrityError):
- self.cu.execute("INSERT OR FAIL INTO test(unique_name) VALUES ('foo')")
- self.assertEqual(self.cu.fetchall(), [])
-
- def test_on_conflict_ignore(self):
- self.cu.execute("INSERT OR IGNORE INTO test(unique_name) VALUES ('foo')")
- # Nothing should happen.
- self.cu.execute("INSERT OR IGNORE INTO test(unique_name) VALUES ('foo')")
- self.cu.execute("SELECT unique_name FROM test")
- self.assertEqual(self.cu.fetchall(), [('foo',)])
-
- def test_on_conflict_replace(self):
- self.cu.execute("INSERT OR REPLACE INTO test(name, unique_name) VALUES ('Data!', 'foo')")
- # There shouldn't be an IntegrityError exception.
- self.cu.execute("INSERT OR REPLACE INTO test(name, unique_name) VALUES ('Very different data!', 'foo')")
- self.cu.execute("SELECT name, unique_name FROM test")
- self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')])
-
-
-class MultiprocessTests(unittest.TestCase):
- CONNECTION_TIMEOUT = SHORT_TIMEOUT / 1000. # Defaults to 30 ms
-
- def tearDown(self):
- unlink(TESTFN)
-
- def test_ctx_mgr_rollback_if_commit_failed(self):
- # bpo-27334: ctx manager does not rollback if commit fails
- SCRIPT = f"""if 1:
- import sqlite3
- def wait():
- print("started")
- assert "database is locked" in input()
-
- cx = sqlite3.connect("{TESTFN}", timeout={self.CONNECTION_TIMEOUT})
- cx.create_function("wait", 0, wait)
- with cx:
- cx.execute("create table t(t)")
- try:
- # execute two transactions; both will try to lock the db
- cx.executescript('''
- -- start a transaction and wait for parent
- begin transaction;
- select * from t;
- select wait();
- rollback;
-
- -- start a new transaction; would fail if parent holds lock
- begin transaction;
- select * from t;
- rollback;
- ''')
- finally:
- cx.close()
- """
-
- # spawn child process
- proc = subprocess.Popen(
- [sys.executable, "-c", SCRIPT],
- encoding="utf-8",
- bufsize=0,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- )
- self.addCleanup(proc.communicate)
-
- # wait for child process to start
- self.assertEqual("started", proc.stdout.readline().strip())
-
- cx = sqlite.connect(TESTFN, timeout=self.CONNECTION_TIMEOUT)
- try: # context manager should correctly release the db lock
- with cx:
- cx.execute("insert into t values('test')")
- except sqlite.OperationalError as exc:
- proc.stdin.write(str(exc))
- else:
- proc.stdin.write("no error")
- finally:
- cx.close()
-
- # terminate child process
- self.assertIsNone(proc.returncode)
- try:
- proc.communicate(input="end", timeout=SHORT_TIMEOUT)
- except subprocess.TimeoutExpired:
- proc.kill()
- proc.communicate()
- raise
- self.assertEqual(proc.returncode, 0)
-
-
-def suite():
- tests = [
- ClosedConTests,
- ClosedCurTests,
- ConnectionTests,
- ConstructorTests,
- CursorTests,
- ExtensionTests,
- ModuleTests,
- MultiprocessTests,
- SqliteOnConflictTests,
- ThreadTests,
- UninitialisedConnectionTests,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sqlite3/test/dump.py b/common/py3-stdlib/sqlite3/test/dump.py
deleted file mode 100644
index 0e324ee..0000000
--- a/common/py3-stdlib/sqlite3/test/dump.py
+++ /dev/null
@@ -1,132 +0,0 @@
-# Author: Paul Kippes <kippesp@gmail.com>
-
-import unittest
-import sqlite3 as sqlite
-
-
-class DumpTests(unittest.TestCase):
- def setUp(self):
- self.cx = sqlite.connect(":memory:")
- self.cu = self.cx.cursor()
-
- def tearDown(self):
- self.cx.close()
-
- def test_table_dump(self):
- expected_sqls = [
- """CREATE TABLE "index"("index" blob);"""
- ,
- """INSERT INTO "index" VALUES(X'01');"""
- ,
- """CREATE TABLE "quoted""table"("quoted""field" text);"""
- ,
- """INSERT INTO "quoted""table" VALUES('quoted''value');"""
- ,
- "CREATE TABLE t1(id integer primary key, s1 text, " \
- "t1_i1 integer not null, i2 integer, unique (s1), " \
- "constraint t1_idx1 unique (i2));"
- ,
- "INSERT INTO \"t1\" VALUES(1,'foo',10,20);"
- ,
- "INSERT INTO \"t1\" VALUES(2,'foo2',30,30);"
- ,
- "CREATE TABLE t2(id integer, t2_i1 integer, " \
- "t2_i2 integer, primary key (id)," \
- "foreign key(t2_i1) references t1(t1_i1));"
- ,
- "CREATE TRIGGER trigger_1 update of t1_i1 on t1 " \
- "begin " \
- "update t2 set t2_i1 = new.t1_i1 where t2_i1 = old.t1_i1; " \
- "end;"
- ,
- "CREATE VIEW v1 as select * from t1 left join t2 " \
- "using (id);"
- ]
- [self.cu.execute(s) for s in expected_sqls]
- i = self.cx.iterdump()
- actual_sqls = [s for s in i]
- expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \
- ['COMMIT;']
- [self.assertEqual(expected_sqls[i], actual_sqls[i])
- for i in range(len(expected_sqls))]
-
- def test_dump_autoincrement(self):
- expected = [
- 'CREATE TABLE "t1" (id integer primary key autoincrement);',
- 'INSERT INTO "t1" VALUES(NULL);',
- 'CREATE TABLE "t2" (id integer primary key autoincrement);',
- ]
- self.cu.executescript("".join(expected))
-
- # the NULL value should now be automatically be set to 1
- expected[1] = expected[1].replace("NULL", "1")
- expected.insert(0, "BEGIN TRANSACTION;")
- expected.extend([
- 'DELETE FROM "sqlite_sequence";',
- 'INSERT INTO "sqlite_sequence" VALUES(\'t1\',1);',
- 'COMMIT;',
- ])
-
- actual = [stmt for stmt in self.cx.iterdump()]
- self.assertEqual(expected, actual)
-
- def test_dump_autoincrement_create_new_db(self):
- self.cu.execute("BEGIN TRANSACTION")
- self.cu.execute("CREATE TABLE t1 (id integer primary key autoincrement)")
- self.cu.execute("CREATE TABLE t2 (id integer primary key autoincrement)")
- self.cu.executemany("INSERT INTO t1 VALUES(?)", ((None,) for _ in range(9)))
- self.cu.executemany("INSERT INTO t2 VALUES(?)", ((None,) for _ in range(4)))
- self.cx.commit()
-
- cx2 = sqlite.connect(":memory:")
- query = "".join(self.cx.iterdump())
- cx2.executescript(query)
- cu2 = cx2.cursor()
-
- dataset = (
- ("t1", 9),
- ("t2", 4),
- )
- for table, seq in dataset:
- with self.subTest(table=table, seq=seq):
- res = cu2.execute("""
- SELECT "seq" FROM "sqlite_sequence" WHERE "name" == ?
- """, (table,))
- rows = res.fetchall()
- self.assertEqual(rows[0][0], seq)
-
- def test_unorderable_row(self):
- # iterdump() should be able to cope with unorderable row types (issue #15545)
- class UnorderableRow:
- def __init__(self, cursor, row):
- self.row = row
- def __getitem__(self, index):
- return self.row[index]
- self.cx.row_factory = UnorderableRow
- CREATE_ALPHA = """CREATE TABLE "alpha" ("one");"""
- CREATE_BETA = """CREATE TABLE "beta" ("two");"""
- expected = [
- "BEGIN TRANSACTION;",
- CREATE_ALPHA,
- CREATE_BETA,
- "COMMIT;"
- ]
- self.cu.execute(CREATE_BETA)
- self.cu.execute(CREATE_ALPHA)
- got = list(self.cx.iterdump())
- self.assertEqual(expected, got)
-
-def suite():
- tests = [
- DumpTests,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sqlite3/test/factory.py b/common/py3-stdlib/sqlite3/test/factory.py
deleted file mode 100644
index 40a290f..0000000
--- a/common/py3-stdlib/sqlite3/test/factory.py
+++ /dev/null
@@ -1,328 +0,0 @@
-# pysqlite2/test/factory.py: tests for the various factories in pysqlite
-#
-# Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
-#
-# This file is part of pysqlite.
-#
-# This software is provided 'as-is', without any express or implied
-# warranty. In no event will the authors be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-# claim that you wrote the original software. If you use this software
-# in a product, an acknowledgment in the product documentation would be
-# appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-# misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-
-import unittest
-import sqlite3 as sqlite
-from collections.abc import Sequence
-
-class MyConnection(sqlite.Connection):
- def __init__(self, *args, **kwargs):
- sqlite.Connection.__init__(self, *args, **kwargs)
-
-def dict_factory(cursor, row):
- d = {}
- for idx, col in enumerate(cursor.description):
- d[col[0]] = row[idx]
- return d
-
-class MyCursor(sqlite.Cursor):
- def __init__(self, *args, **kwargs):
- sqlite.Cursor.__init__(self, *args, **kwargs)
- self.row_factory = dict_factory
-
-class ConnectionFactoryTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:", factory=MyConnection)
-
- def tearDown(self):
- self.con.close()
-
- def test_is_instance(self):
- self.assertIsInstance(self.con, MyConnection)
-
-class CursorFactoryTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
-
- def tearDown(self):
- self.con.close()
-
- def test_is_instance(self):
- cur = self.con.cursor()
- self.assertIsInstance(cur, sqlite.Cursor)
- cur = self.con.cursor(MyCursor)
- self.assertIsInstance(cur, MyCursor)
- cur = self.con.cursor(factory=lambda con: MyCursor(con))
- self.assertIsInstance(cur, MyCursor)
-
- def test_invalid_factory(self):
- # not a callable at all
- self.assertRaises(TypeError, self.con.cursor, None)
- # invalid callable with not exact one argument
- self.assertRaises(TypeError, self.con.cursor, lambda: None)
- # invalid callable returning non-cursor
- self.assertRaises(TypeError, self.con.cursor, lambda con: None)
-
-class RowFactoryTestsBackwardsCompat(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
-
- def test_is_produced_by_factory(self):
- cur = self.con.cursor(factory=MyCursor)
- cur.execute("select 4+5 as foo")
- row = cur.fetchone()
- self.assertIsInstance(row, dict)
- cur.close()
-
- def tearDown(self):
- self.con.close()
-
-class RowFactoryTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
-
- def test_custom_factory(self):
- self.con.row_factory = lambda cur, row: list(row)
- row = self.con.execute("select 1, 2").fetchone()
- self.assertIsInstance(row, list)
-
- def test_sqlite_row_index(self):
- self.con.row_factory = sqlite.Row
- row = self.con.execute("select 1 as a_1, 2 as b").fetchone()
- self.assertIsInstance(row, sqlite.Row)
-
- self.assertEqual(row["a_1"], 1, "by name: wrong result for column 'a_1'")
- self.assertEqual(row["b"], 2, "by name: wrong result for column 'b'")
-
- self.assertEqual(row["A_1"], 1, "by name: wrong result for column 'A_1'")
- self.assertEqual(row["B"], 2, "by name: wrong result for column 'B'")
-
- self.assertEqual(row[0], 1, "by index: wrong result for column 0")
- self.assertEqual(row[1], 2, "by index: wrong result for column 1")
- self.assertEqual(row[-1], 2, "by index: wrong result for column -1")
- self.assertEqual(row[-2], 1, "by index: wrong result for column -2")
-
- with self.assertRaises(IndexError):
- row['c']
- with self.assertRaises(IndexError):
- row['a_\x11']
- with self.assertRaises(IndexError):
- row['a\x7f1']
- with self.assertRaises(IndexError):
- row[2]
- with self.assertRaises(IndexError):
- row[-3]
- with self.assertRaises(IndexError):
- row[2**1000]
-
- def test_sqlite_row_index_unicode(self):
- self.con.row_factory = sqlite.Row
- row = self.con.execute("select 1 as \xff").fetchone()
- self.assertEqual(row["\xff"], 1)
- with self.assertRaises(IndexError):
- row['\u0178']
- with self.assertRaises(IndexError):
- row['\xdf']
-
- def test_sqlite_row_slice(self):
- # A sqlite.Row can be sliced like a list.
- self.con.row_factory = sqlite.Row
- row = self.con.execute("select 1, 2, 3, 4").fetchone()
- self.assertEqual(row[0:0], ())
- self.assertEqual(row[0:1], (1,))
- self.assertEqual(row[1:3], (2, 3))
- self.assertEqual(row[3:1], ())
- # Explicit bounds are optional.
- self.assertEqual(row[1:], (2, 3, 4))
- self.assertEqual(row[:3], (1, 2, 3))
- # Slices can use negative indices.
- self.assertEqual(row[-2:-1], (3,))
- self.assertEqual(row[-2:], (3, 4))
- # Slicing supports steps.
- self.assertEqual(row[0:4:2], (1, 3))
- self.assertEqual(row[3:0:-2], (4, 2))
-
- def test_sqlite_row_iter(self):
- """Checks if the row object is iterable"""
- self.con.row_factory = sqlite.Row
- row = self.con.execute("select 1 as a, 2 as b").fetchone()
-
- # Is iterable in correct order and produces valid results:
- items = [col for col in row]
- self.assertEqual(items, [1, 2])
-
- # Is iterable the second time:
- items = [col for col in row]
- self.assertEqual(items, [1, 2])
-
- def test_sqlite_row_as_tuple(self):
- """Checks if the row object can be converted to a tuple"""
- self.con.row_factory = sqlite.Row
- row = self.con.execute("select 1 as a, 2 as b").fetchone()
- t = tuple(row)
- self.assertEqual(t, (row['a'], row['b']))
-
- def test_sqlite_row_as_dict(self):
- """Checks if the row object can be correctly converted to a dictionary"""
- self.con.row_factory = sqlite.Row
- row = self.con.execute("select 1 as a, 2 as b").fetchone()
- d = dict(row)
- self.assertEqual(d["a"], row["a"])
- self.assertEqual(d["b"], row["b"])
-
- def test_sqlite_row_hash_cmp(self):
- """Checks if the row object compares and hashes correctly"""
- self.con.row_factory = sqlite.Row
- row_1 = self.con.execute("select 1 as a, 2 as b").fetchone()
- row_2 = self.con.execute("select 1 as a, 2 as b").fetchone()
- row_3 = self.con.execute("select 1 as a, 3 as b").fetchone()
- row_4 = self.con.execute("select 1 as b, 2 as a").fetchone()
- row_5 = self.con.execute("select 2 as b, 1 as a").fetchone()
-
- self.assertTrue(row_1 == row_1)
- self.assertTrue(row_1 == row_2)
- self.assertFalse(row_1 == row_3)
- self.assertFalse(row_1 == row_4)
- self.assertFalse(row_1 == row_5)
- self.assertFalse(row_1 == object())
-
- self.assertFalse(row_1 != row_1)
- self.assertFalse(row_1 != row_2)
- self.assertTrue(row_1 != row_3)
- self.assertTrue(row_1 != row_4)
- self.assertTrue(row_1 != row_5)
- self.assertTrue(row_1 != object())
-
- with self.assertRaises(TypeError):
- row_1 > row_2
- with self.assertRaises(TypeError):
- row_1 < row_2
- with self.assertRaises(TypeError):
- row_1 >= row_2
- with self.assertRaises(TypeError):
- row_1 <= row_2
-
- self.assertEqual(hash(row_1), hash(row_2))
-
- def test_sqlite_row_as_sequence(self):
- """ Checks if the row object can act like a sequence """
- self.con.row_factory = sqlite.Row
- row = self.con.execute("select 1 as a, 2 as b").fetchone()
-
- as_tuple = tuple(row)
- self.assertEqual(list(reversed(row)), list(reversed(as_tuple)))
- self.assertIsInstance(row, Sequence)
-
- def test_fake_cursor_class(self):
- # Issue #24257: Incorrect use of PyObject_IsInstance() caused
- # segmentation fault.
- # Issue #27861: Also applies for cursor factory.
- class FakeCursor(str):
- __class__ = sqlite.Cursor
- self.con.row_factory = sqlite.Row
- self.assertRaises(TypeError, self.con.cursor, FakeCursor)
- self.assertRaises(TypeError, sqlite.Row, FakeCursor(), ())
-
- def tearDown(self):
- self.con.close()
-
-class TextFactoryTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
-
- def test_unicode(self):
- austria = "Österreich"
- row = self.con.execute("select ?", (austria,)).fetchone()
- self.assertEqual(type(row[0]), str, "type of row[0] must be unicode")
-
- def test_string(self):
- self.con.text_factory = bytes
- austria = "Österreich"
- row = self.con.execute("select ?", (austria,)).fetchone()
- self.assertEqual(type(row[0]), bytes, "type of row[0] must be bytes")
- self.assertEqual(row[0], austria.encode("utf-8"), "column must equal original data in UTF-8")
-
- def test_custom(self):
- self.con.text_factory = lambda x: str(x, "utf-8", "ignore")
- austria = "Österreich"
- row = self.con.execute("select ?", (austria,)).fetchone()
- self.assertEqual(type(row[0]), str, "type of row[0] must be unicode")
- self.assertTrue(row[0].endswith("reich"), "column must contain original data")
-
- def test_optimized_unicode(self):
- # OptimizedUnicode is deprecated as of Python 3.10
- with self.assertWarns(DeprecationWarning) as cm:
- self.con.text_factory = sqlite.OptimizedUnicode
- self.assertIn("factory.py", cm.filename)
- austria = "Österreich"
- germany = "Deutchland"
- a_row = self.con.execute("select ?", (austria,)).fetchone()
- d_row = self.con.execute("select ?", (germany,)).fetchone()
- self.assertEqual(type(a_row[0]), str, "type of non-ASCII row must be str")
- self.assertEqual(type(d_row[0]), str, "type of ASCII-only row must be str")
-
- def tearDown(self):
- self.con.close()
-
-class TextFactoryTestsWithEmbeddedZeroBytes(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- self.con.execute("create table test (value text)")
- self.con.execute("insert into test (value) values (?)", ("a\x00b",))
-
- def test_string(self):
- # text_factory defaults to str
- row = self.con.execute("select value from test").fetchone()
- self.assertIs(type(row[0]), str)
- self.assertEqual(row[0], "a\x00b")
-
- def test_bytes(self):
- self.con.text_factory = bytes
- row = self.con.execute("select value from test").fetchone()
- self.assertIs(type(row[0]), bytes)
- self.assertEqual(row[0], b"a\x00b")
-
- def test_bytearray(self):
- self.con.text_factory = bytearray
- row = self.con.execute("select value from test").fetchone()
- self.assertIs(type(row[0]), bytearray)
- self.assertEqual(row[0], b"a\x00b")
-
- def test_custom(self):
- # A custom factory should receive a bytes argument
- self.con.text_factory = lambda x: x
- row = self.con.execute("select value from test").fetchone()
- self.assertIs(type(row[0]), bytes)
- self.assertEqual(row[0], b"a\x00b")
-
- def tearDown(self):
- self.con.close()
-
-def suite():
- tests = [
- ConnectionFactoryTests,
- CursorFactoryTests,
- RowFactoryTests,
- RowFactoryTestsBackwardsCompat,
- TextFactoryTests,
- TextFactoryTestsWithEmbeddedZeroBytes,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sqlite3/test/hooks.py b/common/py3-stdlib/sqlite3/test/hooks.py
deleted file mode 100644
index 97121ee..0000000
--- a/common/py3-stdlib/sqlite3/test/hooks.py
+++ /dev/null
@@ -1,306 +0,0 @@
-# pysqlite2/test/hooks.py: tests for various SQLite-specific hooks
-#
-# Copyright (C) 2006-2007 Gerhard Häring <gh@ghaering.de>
-#
-# This file is part of pysqlite.
-#
-# This software is provided 'as-is', without any express or implied
-# warranty. In no event will the authors be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-# claim that you wrote the original software. If you use this software
-# in a product, an acknowledgment in the product documentation would be
-# appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-# misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-
-import contextlib
-import unittest
-import sqlite3 as sqlite
-
-from test.support.os_helper import TESTFN, unlink
-
-
-class CollationTests(unittest.TestCase):
- def test_create_collation_not_string(self):
- con = sqlite.connect(":memory:")
- with self.assertRaises(TypeError):
- con.create_collation(None, lambda x, y: (x > y) - (x < y))
-
- def test_create_collation_not_callable(self):
- con = sqlite.connect(":memory:")
- with self.assertRaises(TypeError) as cm:
- con.create_collation("X", 42)
- self.assertEqual(str(cm.exception), 'parameter must be callable')
-
- def test_create_collation_not_ascii(self):
- con = sqlite.connect(":memory:")
- with self.assertRaises(sqlite.ProgrammingError):
- con.create_collation("collä", lambda x, y: (x > y) - (x < y))
-
- def test_create_collation_bad_upper(self):
- class BadUpperStr(str):
- def upper(self):
- return None
- con = sqlite.connect(":memory:")
- mycoll = lambda x, y: -((x > y) - (x < y))
- con.create_collation(BadUpperStr("mycoll"), mycoll)
- result = con.execute("""
- select x from (
- select 'a' as x
- union
- select 'b' as x
- ) order by x collate mycoll
- """).fetchall()
- self.assertEqual(result[0][0], 'b')
- self.assertEqual(result[1][0], 'a')
-
- def test_collation_is_used(self):
- def mycoll(x, y):
- # reverse order
- return -((x > y) - (x < y))
-
- con = sqlite.connect(":memory:")
- con.create_collation("mycoll", mycoll)
- sql = """
- select x from (
- select 'a' as x
- union
- select 'b' as x
- union
- select 'c' as x
- ) order by x collate mycoll
- """
- result = con.execute(sql).fetchall()
- self.assertEqual(result, [('c',), ('b',), ('a',)],
- msg='the expected order was not returned')
-
- con.create_collation("mycoll", None)
- with self.assertRaises(sqlite.OperationalError) as cm:
- result = con.execute(sql).fetchall()
- self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll')
-
- def test_collation_returns_large_integer(self):
- def mycoll(x, y):
- # reverse order
- return -((x > y) - (x < y)) * 2**32
- con = sqlite.connect(":memory:")
- con.create_collation("mycoll", mycoll)
- sql = """
- select x from (
- select 'a' as x
- union
- select 'b' as x
- union
- select 'c' as x
- ) order by x collate mycoll
- """
- result = con.execute(sql).fetchall()
- self.assertEqual(result, [('c',), ('b',), ('a',)],
- msg="the expected order was not returned")
-
- def test_collation_register_twice(self):
- """
- Register two different collation functions under the same name.
- Verify that the last one is actually used.
- """
- con = sqlite.connect(":memory:")
- con.create_collation("mycoll", lambda x, y: (x > y) - (x < y))
- con.create_collation("mycoll", lambda x, y: -((x > y) - (x < y)))
- result = con.execute("""
- select x from (select 'a' as x union select 'b' as x) order by x collate mycoll
- """).fetchall()
- self.assertEqual(result[0][0], 'b')
- self.assertEqual(result[1][0], 'a')
-
- def test_deregister_collation(self):
- """
- Register a collation, then deregister it. Make sure an error is raised if we try
- to use it.
- """
- con = sqlite.connect(":memory:")
- con.create_collation("mycoll", lambda x, y: (x > y) - (x < y))
- con.create_collation("mycoll", None)
- with self.assertRaises(sqlite.OperationalError) as cm:
- con.execute("select 'a' as x union select 'b' as x order by x collate mycoll")
- self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll')
-
-class ProgressTests(unittest.TestCase):
- def test_progress_handler_used(self):
- """
- Test that the progress handler is invoked once it is set.
- """
- con = sqlite.connect(":memory:")
- progress_calls = []
- def progress():
- progress_calls.append(None)
- return 0
- con.set_progress_handler(progress, 1)
- con.execute("""
- create table foo(a, b)
- """)
- self.assertTrue(progress_calls)
-
-
- def test_opcode_count(self):
- """
- Test that the opcode argument is respected.
- """
- con = sqlite.connect(":memory:")
- progress_calls = []
- def progress():
- progress_calls.append(None)
- return 0
- con.set_progress_handler(progress, 1)
- curs = con.cursor()
- curs.execute("""
- create table foo (a, b)
- """)
- first_count = len(progress_calls)
- progress_calls = []
- con.set_progress_handler(progress, 2)
- curs.execute("""
- create table bar (a, b)
- """)
- second_count = len(progress_calls)
- self.assertGreaterEqual(first_count, second_count)
-
- def test_cancel_operation(self):
- """
- Test that returning a non-zero value stops the operation in progress.
- """
- con = sqlite.connect(":memory:")
- def progress():
- return 1
- con.set_progress_handler(progress, 1)
- curs = con.cursor()
- self.assertRaises(
- sqlite.OperationalError,
- curs.execute,
- "create table bar (a, b)")
-
- def test_clear_handler(self):
- """
- Test that setting the progress handler to None clears the previously set handler.
- """
- con = sqlite.connect(":memory:")
- action = 0
- def progress():
- nonlocal action
- action = 1
- return 0
- con.set_progress_handler(progress, 1)
- con.set_progress_handler(None, 1)
- con.execute("select 1 union select 2 union select 3").fetchall()
- self.assertEqual(action, 0, "progress handler was not cleared")
-
-class TraceCallbackTests(unittest.TestCase):
- @contextlib.contextmanager
- def check_stmt_trace(self, cx, expected):
- try:
- traced = []
- cx.set_trace_callback(lambda stmt: traced.append(stmt))
- yield
- finally:
- self.assertEqual(traced, expected)
- cx.set_trace_callback(None)
-
- def test_trace_callback_used(self):
- """
- Test that the trace callback is invoked once it is set.
- """
- con = sqlite.connect(":memory:")
- traced_statements = []
- def trace(statement):
- traced_statements.append(statement)
- con.set_trace_callback(trace)
- con.execute("create table foo(a, b)")
- self.assertTrue(traced_statements)
- self.assertTrue(any("create table foo" in stmt for stmt in traced_statements))
-
- def test_clear_trace_callback(self):
- """
- Test that setting the trace callback to None clears the previously set callback.
- """
- con = sqlite.connect(":memory:")
- traced_statements = []
- def trace(statement):
- traced_statements.append(statement)
- con.set_trace_callback(trace)
- con.set_trace_callback(None)
- con.execute("create table foo(a, b)")
- self.assertFalse(traced_statements, "trace callback was not cleared")
-
- def test_unicode_content(self):
- """
- Test that the statement can contain unicode literals.
- """
- unicode_value = '\xf6\xe4\xfc\xd6\xc4\xdc\xdf\u20ac'
- con = sqlite.connect(":memory:")
- traced_statements = []
- def trace(statement):
- traced_statements.append(statement)
- con.set_trace_callback(trace)
- con.execute("create table foo(x)")
- con.execute("insert into foo(x) values ('%s')" % unicode_value)
- con.commit()
- self.assertTrue(any(unicode_value in stmt for stmt in traced_statements),
- "Unicode data %s garbled in trace callback: %s"
- % (ascii(unicode_value), ', '.join(map(ascii, traced_statements))))
-
- def test_trace_callback_content(self):
- # set_trace_callback() shouldn't produce duplicate content (bpo-26187)
- traced_statements = []
- def trace(statement):
- traced_statements.append(statement)
-
- queries = ["create table foo(x)",
- "insert into foo(x) values(1)"]
- self.addCleanup(unlink, TESTFN)
- con1 = sqlite.connect(TESTFN, isolation_level=None)
- con2 = sqlite.connect(TESTFN)
- con1.set_trace_callback(trace)
- cur = con1.cursor()
- cur.execute(queries[0])
- con2.execute("create table bar(x)")
- cur.execute(queries[1])
- self.assertEqual(traced_statements, queries)
-
- def test_trace_expanded_sql(self):
- expected = [
- "create table t(t)",
- "BEGIN ",
- "insert into t values(0)",
- "insert into t values(1)",
- "insert into t values(2)",
- "COMMIT",
- ]
- cx = sqlite.connect(":memory:")
- with self.check_stmt_trace(cx, expected):
- with cx:
- cx.execute("create table t(t)")
- cx.executemany("insert into t values(?)", ((v,) for v in range(3)))
-
-
-def suite():
- tests = [
- CollationTests,
- ProgressTests,
- TraceCallbackTests,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sqlite3/test/regression.py b/common/py3-stdlib/sqlite3/test/regression.py
deleted file mode 100644
index 8937f9f..0000000
--- a/common/py3-stdlib/sqlite3/test/regression.py
+++ /dev/null
@@ -1,471 +0,0 @@
-# pysqlite2/test/regression.py: pysqlite regression tests
-#
-# Copyright (C) 2006-2010 Gerhard Häring <gh@ghaering.de>
-#
-# This file is part of pysqlite.
-#
-# This software is provided 'as-is', without any express or implied
-# warranty. In no event will the authors be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-# claim that you wrote the original software. If you use this software
-# in a product, an acknowledgment in the product documentation would be
-# appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-# misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-
-import datetime
-import unittest
-import sqlite3 as sqlite
-import weakref
-import functools
-from test import support
-
-from unittest.mock import patch
-
-
-class RegressionTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
-
- def tearDown(self):
- self.con.close()
-
- def test_pragma_user_version(self):
- # This used to crash pysqlite because this pragma command returns NULL for the column name
- cur = self.con.cursor()
- cur.execute("pragma user_version")
-
- def test_pragma_schema_version(self):
- # This still crashed pysqlite <= 2.2.1
- con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES)
- try:
- cur = self.con.cursor()
- cur.execute("pragma schema_version")
- finally:
- cur.close()
- con.close()
-
- def test_statement_reset(self):
- # pysqlite 2.1.0 to 2.2.0 have the problem that not all statements are
- # reset before a rollback, but only those that are still in the
- # statement cache. The others are not accessible from the connection object.
- con = sqlite.connect(":memory:", cached_statements=5)
- cursors = [con.cursor() for x in range(5)]
- cursors[0].execute("create table test(x)")
- for i in range(10):
- cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)])
-
- for i in range(5):
- cursors[i].execute(" " * i + "select x from test")
-
- con.rollback()
-
- def test_column_name_with_spaces(self):
- cur = self.con.cursor()
- cur.execute('select 1 as "foo bar [datetime]"')
- self.assertEqual(cur.description[0][0], "foo bar [datetime]")
-
- cur.execute('select 1 as "foo baz"')
- self.assertEqual(cur.description[0][0], "foo baz")
-
- def test_statement_finalization_on_close_db(self):
- # pysqlite versions <= 2.3.3 only finalized statements in the statement
- # cache when closing the database. statements that were still
- # referenced in cursors weren't closed and could provoke "
- # "OperationalError: Unable to close due to unfinalised statements".
- con = sqlite.connect(":memory:")
- cursors = []
- # default statement cache size is 100
- for i in range(105):
- cur = con.cursor()
- cursors.append(cur)
- cur.execute("select 1 x union select " + str(i))
- con.close()
-
- def test_on_conflict_rollback(self):
- con = sqlite.connect(":memory:")
- con.execute("create table foo(x, unique(x) on conflict rollback)")
- con.execute("insert into foo(x) values (1)")
- try:
- con.execute("insert into foo(x) values (1)")
- except sqlite.DatabaseError:
- pass
- con.execute("insert into foo(x) values (2)")
- try:
- con.commit()
- except sqlite.OperationalError:
- self.fail("pysqlite knew nothing about the implicit ROLLBACK")
-
- def test_workaround_for_buggy_sqlite_transfer_bindings(self):
- """
- pysqlite would crash with older SQLite versions unless
- a workaround is implemented.
- """
- self.con.execute("create table foo(bar)")
- self.con.execute("drop table foo")
- self.con.execute("create table foo(bar)")
-
- def test_empty_statement(self):
- """
- pysqlite used to segfault with SQLite versions 3.5.x. These return NULL
- for "no-operation" statements
- """
- self.con.execute("")
-
- def test_type_map_usage(self):
- """
- pysqlite until 2.4.1 did not rebuild the row_cast_map when recompiling
- a statement. This test exhibits the problem.
- """
- SELECT = "select * from foo"
- con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES)
- con.execute("create table foo(bar timestamp)")
- con.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),))
- con.execute(SELECT).close()
- con.execute("drop table foo")
- con.execute("create table foo(bar integer)")
- con.execute("insert into foo(bar) values (5)")
- con.execute(SELECT).close()
-
- def test_bind_mutating_list(self):
- # Issue41662: Crash when mutate a list of parameters during iteration.
- class X:
- def __conform__(self, protocol):
- parameters.clear()
- return "..."
- parameters = [X(), 0]
- con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES)
- con.execute("create table foo(bar X, baz integer)")
- # Should not crash
- with self.assertRaises(IndexError):
- con.execute("insert into foo(bar, baz) values (?, ?)", parameters)
-
- def test_error_msg_decode_error(self):
- # When porting the module to Python 3.0, the error message about
- # decoding errors disappeared. This verifies they're back again.
- with self.assertRaises(sqlite.OperationalError) as cm:
- self.con.execute("select 'xxx' || ? || 'yyy' colname",
- (bytes(bytearray([250])),)).fetchone()
- msg = "Could not decode to UTF-8 column 'colname' with text 'xxx"
- self.assertIn(msg, str(cm.exception))
-
- def test_register_adapter(self):
- """
- See issue 3312.
- """
- self.assertRaises(TypeError, sqlite.register_adapter, {}, None)
-
- def test_set_isolation_level(self):
- # See issue 27881.
- class CustomStr(str):
- def upper(self):
- return None
- def __del__(self):
- con.isolation_level = ""
-
- con = sqlite.connect(":memory:")
- con.isolation_level = None
- for level in "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE":
- with self.subTest(level=level):
- con.isolation_level = level
- con.isolation_level = level.lower()
- con.isolation_level = level.capitalize()
- con.isolation_level = CustomStr(level)
-
- # setting isolation_level failure should not alter previous state
- con.isolation_level = None
- con.isolation_level = "DEFERRED"
- pairs = [
- (1, TypeError), (b'', TypeError), ("abc", ValueError),
- ("IMMEDIATE\0EXCLUSIVE", ValueError), ("\xe9", ValueError),
- ]
- for value, exc in pairs:
- with self.subTest(level=value):
- with self.assertRaises(exc):
- con.isolation_level = value
- self.assertEqual(con.isolation_level, "DEFERRED")
-
- def test_cursor_constructor_call_check(self):
- """
- Verifies that cursor methods check whether base class __init__ was
- called.
- """
- class Cursor(sqlite.Cursor):
- def __init__(self, con):
- pass
-
- con = sqlite.connect(":memory:")
- cur = Cursor(con)
- with self.assertRaises(sqlite.ProgrammingError):
- cur.execute("select 4+5").fetchall()
- with self.assertRaisesRegex(sqlite.ProgrammingError,
- r'^Base Cursor\.__init__ not called\.$'):
- cur.close()
-
- def test_str_subclass(self):
- """
- The Python 3.0 port of the module didn't cope with values of subclasses of str.
- """
- class MyStr(str): pass
- self.con.execute("select ?", (MyStr("abc"),))
-
- def test_connection_constructor_call_check(self):
- """
- Verifies that connection methods check whether base class __init__ was
- called.
- """
- class Connection(sqlite.Connection):
- def __init__(self, name):
- pass
-
- con = Connection(":memory:")
- with self.assertRaises(sqlite.ProgrammingError):
- cur = con.cursor()
-
- def test_cursor_registration(self):
- """
- Verifies that subclassed cursor classes are correctly registered with
- the connection object, too. (fetch-across-rollback problem)
- """
- class Connection(sqlite.Connection):
- def cursor(self):
- return Cursor(self)
-
- class Cursor(sqlite.Cursor):
- def __init__(self, con):
- sqlite.Cursor.__init__(self, con)
-
- con = Connection(":memory:")
- cur = con.cursor()
- cur.execute("create table foo(x)")
- cur.executemany("insert into foo(x) values (?)", [(3,), (4,), (5,)])
- cur.execute("select x from foo")
- con.rollback()
- with self.assertRaises(sqlite.InterfaceError):
- cur.fetchall()
-
- def test_auto_commit(self):
- """
- Verifies that creating a connection in autocommit mode works.
- 2.5.3 introduced a regression so that these could no longer
- be created.
- """
- con = sqlite.connect(":memory:", isolation_level=None)
-
- def test_pragma_autocommit(self):
- """
- Verifies that running a PRAGMA statement that does an autocommit does
- work. This did not work in 2.5.3/2.5.4.
- """
- cur = self.con.cursor()
- cur.execute("create table foo(bar)")
- cur.execute("insert into foo(bar) values (5)")
-
- cur.execute("pragma page_size")
- row = cur.fetchone()
-
- def test_connection_call(self):
- """
- Call a connection with a non-string SQL request: check error handling
- of the statement constructor.
- """
- self.assertRaises(TypeError, self.con, 1)
-
- def test_collation(self):
- def collation_cb(a, b):
- return 1
- self.assertRaises(sqlite.ProgrammingError, self.con.create_collation,
- # Lone surrogate cannot be encoded to the default encoding (utf8)
- "\uDC80", collation_cb)
-
- def test_recursive_cursor_use(self):
- """
- http://bugs.python.org/issue10811
-
- Recursively using a cursor, such as when reusing it from a generator led to segfaults.
- Now we catch recursive cursor usage and raise a ProgrammingError.
- """
- con = sqlite.connect(":memory:")
-
- cur = con.cursor()
- cur.execute("create table a (bar)")
- cur.execute("create table b (baz)")
-
- def foo():
- cur.execute("insert into a (bar) values (?)", (1,))
- yield 1
-
- with self.assertRaises(sqlite.ProgrammingError):
- cur.executemany("insert into b (baz) values (?)",
- ((i,) for i in foo()))
-
- def test_convert_timestamp_microsecond_padding(self):
- """
- http://bugs.python.org/issue14720
-
- The microsecond parsing of convert_timestamp() should pad with zeros,
- since the microsecond string "456" actually represents "456000".
- """
-
- con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES)
- cur = con.cursor()
- cur.execute("CREATE TABLE t (x TIMESTAMP)")
-
- # Microseconds should be 456000
- cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')")
-
- # Microseconds should be truncated to 123456
- cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')")
-
- cur.execute("SELECT * FROM t")
- values = [x[0] for x in cur.fetchall()]
-
- self.assertEqual(values, [
- datetime.datetime(2012, 4, 4, 15, 6, 0, 456000),
- datetime.datetime(2012, 4, 4, 15, 6, 0, 123456),
- ])
-
- def test_invalid_isolation_level_type(self):
- # isolation level is a string, not an integer
- self.assertRaises(TypeError,
- sqlite.connect, ":memory:", isolation_level=123)
-
-
- def test_null_character(self):
- # Issue #21147
- con = sqlite.connect(":memory:")
- self.assertRaises(ValueError, con, "\0select 1")
- self.assertRaises(ValueError, con, "select 1\0")
- cur = con.cursor()
- self.assertRaises(ValueError, cur.execute, " \0select 2")
- self.assertRaises(ValueError, cur.execute, "select 2\0")
-
- def test_commit_cursor_reset(self):
- """
- Connection.commit() did reset cursors, which made sqlite3
- to return rows multiple times when fetched from cursors
- after commit. See issues 10513 and 23129 for details.
- """
- con = sqlite.connect(":memory:")
- con.executescript("""
- create table t(c);
- create table t2(c);
- insert into t values(0);
- insert into t values(1);
- insert into t values(2);
- """)
-
- self.assertEqual(con.isolation_level, "")
-
- counter = 0
- for i, row in enumerate(con.execute("select c from t")):
- with self.subTest(i=i, row=row):
- con.execute("insert into t2(c) values (?)", (i,))
- con.commit()
- if counter == 0:
- self.assertEqual(row[0], 0)
- elif counter == 1:
- self.assertEqual(row[0], 1)
- elif counter == 2:
- self.assertEqual(row[0], 2)
- counter += 1
- self.assertEqual(counter, 3, "should have returned exactly three rows")
-
- def test_bpo31770(self):
- """
- The interpreter shouldn't crash in case Cursor.__init__() is called
- more than once.
- """
- def callback(*args):
- pass
- con = sqlite.connect(":memory:")
- cur = sqlite.Cursor(con)
- ref = weakref.ref(cur, callback)
- cur.__init__(con)
- del cur
- # The interpreter shouldn't crash when ref is collected.
- del ref
- support.gc_collect()
-
- def test_del_isolation_level_segfault(self):
- with self.assertRaises(AttributeError):
- del self.con.isolation_level
-
- def test_bpo37347(self):
- class Printer:
- def log(self, *args):
- return sqlite.SQLITE_OK
-
- for method in [self.con.set_trace_callback,
- functools.partial(self.con.set_progress_handler, n=1),
- self.con.set_authorizer]:
- printer_instance = Printer()
- method(printer_instance.log)
- method(printer_instance.log)
- self.con.execute("select 1") # trigger seg fault
- method(None)
-
- def test_return_empty_bytestring(self):
- cur = self.con.execute("select X''")
- val = cur.fetchone()[0]
- self.assertEqual(val, b'')
-
-
-class RecursiveUseOfCursors(unittest.TestCase):
- # GH-80254: sqlite3 should not segfault for recursive use of cursors.
- msg = "Recursive use of cursors not allowed"
-
- def setUp(self):
- self.con = sqlite.connect(":memory:",
- detect_types=sqlite.PARSE_COLNAMES)
- self.cur = self.con.cursor()
- self.cur.execute("create table test(x foo)")
- self.cur.executemany("insert into test(x) values (?)",
- [("foo",), ("bar",)])
-
- def tearDown(self):
- self.cur.close()
- self.con.close()
-
- def test_recursive_cursor_init(self):
- conv = lambda x: self.cur.__init__(self.con)
- with patch.dict(sqlite.converters, {"INIT": conv}):
- with self.assertRaisesRegex(sqlite.ProgrammingError, self.msg):
- self.cur.execute(f'select x as "x [INIT]", x from test')
-
- def test_recursive_cursor_close(self):
- conv = lambda x: self.cur.close()
- with patch.dict(sqlite.converters, {"CLOSE": conv}):
- with self.assertRaisesRegex(sqlite.ProgrammingError, self.msg):
- self.cur.execute(f'select x as "x [CLOSE]", x from test')
-
- def test_recursive_cursor_fetch(self):
- conv = lambda x, l=[]: self.cur.fetchone() if l else l.append(None)
- with patch.dict(sqlite.converters, {"ITER": conv}):
- self.cur.execute(f'select x as "x [ITER]", x from test')
- with self.assertRaisesRegex(sqlite.ProgrammingError, self.msg):
- self.cur.fetchall()
-
-
-def suite():
- tests = [
- RegressionTests,
- RecursiveUseOfCursors,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sqlite3/test/transactions.py b/common/py3-stdlib/sqlite3/test/transactions.py
deleted file mode 100644
index 4ebc7fb..0000000
--- a/common/py3-stdlib/sqlite3/test/transactions.py
+++ /dev/null
@@ -1,211 +0,0 @@
-# pysqlite2/test/transactions.py: tests transactions
-#
-# Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
-#
-# This file is part of pysqlite.
-#
-# This software is provided 'as-is', without any express or implied
-# warranty. In no event will the authors be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-# claim that you wrote the original software. If you use this software
-# in a product, an acknowledgment in the product documentation would be
-# appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-# misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-
-import os, unittest
-import sqlite3 as sqlite
-
-from test.support import LOOPBACK_TIMEOUT
-from test.support.os_helper import TESTFN, unlink
-
-
-TIMEOUT = LOOPBACK_TIMEOUT / 10
-
-
-class TransactionTests(unittest.TestCase):
- def setUp(self):
- self.con1 = sqlite.connect(TESTFN, timeout=TIMEOUT)
- self.cur1 = self.con1.cursor()
-
- self.con2 = sqlite.connect(TESTFN, timeout=TIMEOUT)
- self.cur2 = self.con2.cursor()
-
- def tearDown(self):
- try:
- self.cur1.close()
- self.con1.close()
-
- self.cur2.close()
- self.con2.close()
-
- finally:
- unlink(TESTFN)
-
- def test_dml_does_not_auto_commit_before(self):
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- self.cur1.execute("create table test2(j)")
- self.cur2.execute("select i from test")
- res = self.cur2.fetchall()
- self.assertEqual(len(res), 0)
-
- def test_insert_starts_transaction(self):
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- self.cur2.execute("select i from test")
- res = self.cur2.fetchall()
- self.assertEqual(len(res), 0)
-
- def test_update_starts_transaction(self):
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- self.con1.commit()
- self.cur1.execute("update test set i=6")
- self.cur2.execute("select i from test")
- res = self.cur2.fetchone()[0]
- self.assertEqual(res, 5)
-
- def test_delete_starts_transaction(self):
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- self.con1.commit()
- self.cur1.execute("delete from test")
- self.cur2.execute("select i from test")
- res = self.cur2.fetchall()
- self.assertEqual(len(res), 1)
-
- def test_replace_starts_transaction(self):
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- self.con1.commit()
- self.cur1.execute("replace into test(i) values (6)")
- self.cur2.execute("select i from test")
- res = self.cur2.fetchall()
- self.assertEqual(len(res), 1)
- self.assertEqual(res[0][0], 5)
-
- def test_toggle_auto_commit(self):
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- self.con1.isolation_level = None
- self.assertEqual(self.con1.isolation_level, None)
- self.cur2.execute("select i from test")
- res = self.cur2.fetchall()
- self.assertEqual(len(res), 1)
-
- self.con1.isolation_level = "DEFERRED"
- self.assertEqual(self.con1.isolation_level , "DEFERRED")
- self.cur1.execute("insert into test(i) values (5)")
- self.cur2.execute("select i from test")
- res = self.cur2.fetchall()
- self.assertEqual(len(res), 1)
-
- def test_raise_timeout(self):
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- with self.assertRaises(sqlite.OperationalError):
- self.cur2.execute("insert into test(i) values (5)")
-
- def test_locking(self):
- """
- This tests the improved concurrency with pysqlite 2.3.4. You needed
- to roll back con2 before you could commit con1.
- """
- self.cur1.execute("create table test(i)")
- self.cur1.execute("insert into test(i) values (5)")
- with self.assertRaises(sqlite.OperationalError):
- self.cur2.execute("insert into test(i) values (5)")
- # NO self.con2.rollback() HERE!!!
- self.con1.commit()
-
- def test_rollback_cursor_consistency(self):
- """
- Checks if cursors on the connection are set into a "reset" state
- when a rollback is done on the connection.
- """
- con = sqlite.connect(":memory:")
- cur = con.cursor()
- cur.execute("create table test(x)")
- cur.execute("insert into test(x) values (5)")
- cur.execute("select 1 union select 2 union select 3")
-
- con.rollback()
- with self.assertRaises(sqlite.InterfaceError):
- cur.fetchall()
-
-class SpecialCommandTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- self.cur = self.con.cursor()
-
- def test_drop_table(self):
- self.cur.execute("create table test(i)")
- self.cur.execute("insert into test(i) values (5)")
- self.cur.execute("drop table test")
-
- def test_pragma(self):
- self.cur.execute("create table test(i)")
- self.cur.execute("insert into test(i) values (5)")
- self.cur.execute("pragma count_changes=1")
-
- def tearDown(self):
- self.cur.close()
- self.con.close()
-
-class TransactionalDDL(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
-
- def test_ddl_does_not_autostart_transaction(self):
- # For backwards compatibility reasons, DDL statements should not
- # implicitly start a transaction.
- self.con.execute("create table test(i)")
- self.con.rollback()
- result = self.con.execute("select * from test").fetchall()
- self.assertEqual(result, [])
-
- def test_immediate_transactional_ddl(self):
- # You can achieve transactional DDL by issuing a BEGIN
- # statement manually.
- self.con.execute("begin immediate")
- self.con.execute("create table test(i)")
- self.con.rollback()
- with self.assertRaises(sqlite.OperationalError):
- self.con.execute("select * from test")
-
- def test_transactional_ddl(self):
- # You can achieve transactional DDL by issuing a BEGIN
- # statement manually.
- self.con.execute("begin")
- self.con.execute("create table test(i)")
- self.con.rollback()
- with self.assertRaises(sqlite.OperationalError):
- self.con.execute("select * from test")
-
- def tearDown(self):
- self.con.close()
-
-def suite():
- tests = [
- SpecialCommandTests,
- TransactionTests,
- TransactionalDDL,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sqlite3/test/types.py b/common/py3-stdlib/sqlite3/test/types.py
deleted file mode 100644
index 4bb1de8..0000000
--- a/common/py3-stdlib/sqlite3/test/types.py
+++ /dev/null
@@ -1,467 +0,0 @@
-# pysqlite2/test/types.py: tests for type conversion and detection
-#
-# Copyright (C) 2005 Gerhard Häring <gh@ghaering.de>
-#
-# This file is part of pysqlite.
-#
-# This software is provided 'as-is', without any express or implied
-# warranty. In no event will the authors be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-# claim that you wrote the original software. If you use this software
-# in a product, an acknowledgment in the product documentation would be
-# appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-# misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-
-import datetime
-import unittest
-import sqlite3 as sqlite
-try:
- import zlib
-except ImportError:
- zlib = None
-
-
-class SqliteTypeTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- self.cur = self.con.cursor()
- self.cur.execute("create table test(i integer, s varchar, f number, b blob)")
-
- def tearDown(self):
- self.cur.close()
- self.con.close()
-
- def test_string(self):
- self.cur.execute("insert into test(s) values (?)", ("Österreich",))
- self.cur.execute("select s from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], "Österreich")
-
- def test_small_int(self):
- self.cur.execute("insert into test(i) values (?)", (42,))
- self.cur.execute("select i from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], 42)
-
- def test_large_int(self):
- num = 2**40
- self.cur.execute("insert into test(i) values (?)", (num,))
- self.cur.execute("select i from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], num)
-
- def test_float(self):
- val = 3.14
- self.cur.execute("insert into test(f) values (?)", (val,))
- self.cur.execute("select f from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], val)
-
- def test_blob(self):
- sample = b"Guglhupf"
- val = memoryview(sample)
- self.cur.execute("insert into test(b) values (?)", (val,))
- self.cur.execute("select b from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], sample)
-
- def test_unicode_execute(self):
- self.cur.execute("select 'Österreich'")
- row = self.cur.fetchone()
- self.assertEqual(row[0], "Österreich")
-
-class DeclTypesTests(unittest.TestCase):
- class Foo:
- def __init__(self, _val):
- if isinstance(_val, bytes):
- # sqlite3 always calls __init__ with a bytes created from a
- # UTF-8 string when __conform__ was used to store the object.
- _val = _val.decode('utf-8')
- self.val = _val
-
- def __eq__(self, other):
- if not isinstance(other, DeclTypesTests.Foo):
- return NotImplemented
- return self.val == other.val
-
- def __conform__(self, protocol):
- if protocol is sqlite.PrepareProtocol:
- return self.val
- else:
- return None
-
- def __str__(self):
- return "<%s>" % self.val
-
- class BadConform:
- def __init__(self, exc):
- self.exc = exc
- def __conform__(self, protocol):
- raise self.exc
-
- def setUp(self):
- self.con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES)
- self.cur = self.con.cursor()
- self.cur.execute("""
- create table test(
- i int,
- s str,
- f float,
- b bool,
- u unicode,
- foo foo,
- bin blob,
- n1 number,
- n2 number(5),
- bad bad,
- cbin cblob)
- """)
-
- # override float, make them always return the same number
- sqlite.converters["FLOAT"] = lambda x: 47.2
-
- # and implement two custom ones
- sqlite.converters["BOOL"] = lambda x: bool(int(x))
- sqlite.converters["FOO"] = DeclTypesTests.Foo
- sqlite.converters["BAD"] = DeclTypesTests.BadConform
- sqlite.converters["WRONG"] = lambda x: "WRONG"
- sqlite.converters["NUMBER"] = float
- sqlite.converters["CBLOB"] = lambda x: b"blobish"
-
- def tearDown(self):
- del sqlite.converters["FLOAT"]
- del sqlite.converters["BOOL"]
- del sqlite.converters["FOO"]
- del sqlite.converters["BAD"]
- del sqlite.converters["WRONG"]
- del sqlite.converters["NUMBER"]
- del sqlite.converters["CBLOB"]
- self.cur.close()
- self.con.close()
-
- def test_string(self):
- # default
- self.cur.execute("insert into test(s) values (?)", ("foo",))
- self.cur.execute('select s as "s [WRONG]" from test')
- row = self.cur.fetchone()
- self.assertEqual(row[0], "foo")
-
- def test_small_int(self):
- # default
- self.cur.execute("insert into test(i) values (?)", (42,))
- self.cur.execute("select i from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], 42)
-
- def test_large_int(self):
- # default
- num = 2**40
- self.cur.execute("insert into test(i) values (?)", (num,))
- self.cur.execute("select i from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], num)
-
- def test_float(self):
- # custom
- val = 3.14
- self.cur.execute("insert into test(f) values (?)", (val,))
- self.cur.execute("select f from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], 47.2)
-
- def test_bool(self):
- # custom
- self.cur.execute("insert into test(b) values (?)", (False,))
- self.cur.execute("select b from test")
- row = self.cur.fetchone()
- self.assertIs(row[0], False)
-
- self.cur.execute("delete from test")
- self.cur.execute("insert into test(b) values (?)", (True,))
- self.cur.execute("select b from test")
- row = self.cur.fetchone()
- self.assertIs(row[0], True)
-
- def test_unicode(self):
- # default
- val = "\xd6sterreich"
- self.cur.execute("insert into test(u) values (?)", (val,))
- self.cur.execute("select u from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], val)
-
- def test_foo(self):
- val = DeclTypesTests.Foo("bla")
- self.cur.execute("insert into test(foo) values (?)", (val,))
- self.cur.execute("select foo from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], val)
-
- def test_error_in_conform(self):
- val = DeclTypesTests.BadConform(TypeError)
- with self.assertRaises(sqlite.InterfaceError):
- self.cur.execute("insert into test(bad) values (?)", (val,))
- with self.assertRaises(sqlite.InterfaceError):
- self.cur.execute("insert into test(bad) values (:val)", {"val": val})
-
- val = DeclTypesTests.BadConform(KeyboardInterrupt)
- with self.assertRaises(KeyboardInterrupt):
- self.cur.execute("insert into test(bad) values (?)", (val,))
- with self.assertRaises(KeyboardInterrupt):
- self.cur.execute("insert into test(bad) values (:val)", {"val": val})
-
- def test_unsupported_seq(self):
- class Bar: pass
- val = Bar()
- with self.assertRaises(sqlite.InterfaceError):
- self.cur.execute("insert into test(f) values (?)", (val,))
-
- def test_unsupported_dict(self):
- class Bar: pass
- val = Bar()
- with self.assertRaises(sqlite.InterfaceError):
- self.cur.execute("insert into test(f) values (:val)", {"val": val})
-
- def test_blob(self):
- # default
- sample = b"Guglhupf"
- val = memoryview(sample)
- self.cur.execute("insert into test(bin) values (?)", (val,))
- self.cur.execute("select bin from test")
- row = self.cur.fetchone()
- self.assertEqual(row[0], sample)
-
- def test_number1(self):
- self.cur.execute("insert into test(n1) values (5)")
- value = self.cur.execute("select n1 from test").fetchone()[0]
- # if the converter is not used, it's an int instead of a float
- self.assertEqual(type(value), float)
-
- def test_number2(self):
- """Checks whether converter names are cut off at '(' characters"""
- self.cur.execute("insert into test(n2) values (5)")
- value = self.cur.execute("select n2 from test").fetchone()[0]
- # if the converter is not used, it's an int instead of a float
- self.assertEqual(type(value), float)
-
- def test_convert_zero_sized_blob(self):
- self.con.execute("insert into test(cbin) values (?)", (b"",))
- cur = self.con.execute("select cbin from test")
- # Zero-sized blobs with converters returns None. This differs from
- # blobs without a converter, where b"" is returned.
- self.assertIsNone(cur.fetchone()[0])
-
-
-class ColNamesTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES)
- self.cur = self.con.cursor()
- self.cur.execute("create table test(x foo)")
-
- sqlite.converters["FOO"] = lambda x: "[%s]" % x.decode("ascii")
- sqlite.converters["BAR"] = lambda x: "<%s>" % x.decode("ascii")
- sqlite.converters["EXC"] = lambda x: 5/0
- sqlite.converters["B1B1"] = lambda x: "MARKER"
-
- def tearDown(self):
- del sqlite.converters["FOO"]
- del sqlite.converters["BAR"]
- del sqlite.converters["EXC"]
- del sqlite.converters["B1B1"]
- self.cur.close()
- self.con.close()
-
- def test_decl_type_not_used(self):
- """
- Assures that the declared type is not used when PARSE_DECLTYPES
- is not set.
- """
- self.cur.execute("insert into test(x) values (?)", ("xxx",))
- self.cur.execute("select x from test")
- val = self.cur.fetchone()[0]
- self.assertEqual(val, "xxx")
-
- def test_none(self):
- self.cur.execute("insert into test(x) values (?)", (None,))
- self.cur.execute("select x from test")
- val = self.cur.fetchone()[0]
- self.assertEqual(val, None)
-
- def test_col_name(self):
- self.cur.execute("insert into test(x) values (?)", ("xxx",))
- self.cur.execute('select x as "x y [bar]" from test')
- val = self.cur.fetchone()[0]
- self.assertEqual(val, "<xxx>")
-
- # Check if the stripping of colnames works. Everything after the first
- # '[' (and the preceeding space) should be stripped.
- self.assertEqual(self.cur.description[0][0], "x y")
-
- def test_case_in_converter_name(self):
- self.cur.execute("select 'other' as \"x [b1b1]\"")
- val = self.cur.fetchone()[0]
- self.assertEqual(val, "MARKER")
-
- def test_cursor_description_no_row(self):
- """
- cursor.description should at least provide the column name(s), even if
- no row returned.
- """
- self.cur.execute("select * from test where 0 = 1")
- self.assertEqual(self.cur.description[0][0], "x")
-
- def test_cursor_description_insert(self):
- self.cur.execute("insert into test values (1)")
- self.assertIsNone(self.cur.description)
-
-
-@unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "CTEs not supported")
-class CommonTableExpressionTests(unittest.TestCase):
-
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- self.cur = self.con.cursor()
- self.cur.execute("create table test(x foo)")
-
- def tearDown(self):
- self.cur.close()
- self.con.close()
-
- def test_cursor_description_cte_simple(self):
- self.cur.execute("with one as (select 1) select * from one")
- self.assertIsNotNone(self.cur.description)
- self.assertEqual(self.cur.description[0][0], "1")
-
- def test_cursor_description_cte_multiple_columns(self):
- self.cur.execute("insert into test values(1)")
- self.cur.execute("insert into test values(2)")
- self.cur.execute("with testCTE as (select * from test) select * from testCTE")
- self.assertIsNotNone(self.cur.description)
- self.assertEqual(self.cur.description[0][0], "x")
-
- def test_cursor_description_cte(self):
- self.cur.execute("insert into test values (1)")
- self.cur.execute("with bar as (select * from test) select * from test where x = 1")
- self.assertIsNotNone(self.cur.description)
- self.assertEqual(self.cur.description[0][0], "x")
- self.cur.execute("with bar as (select * from test) select * from test where x = 2")
- self.assertIsNotNone(self.cur.description)
- self.assertEqual(self.cur.description[0][0], "x")
-
-
-class ObjectAdaptationTests(unittest.TestCase):
- def cast(obj):
- return float(obj)
- cast = staticmethod(cast)
-
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- try:
- del sqlite.adapters[int]
- except:
- pass
- sqlite.register_adapter(int, ObjectAdaptationTests.cast)
- self.cur = self.con.cursor()
-
- def tearDown(self):
- del sqlite.adapters[(int, sqlite.PrepareProtocol)]
- self.cur.close()
- self.con.close()
-
- def test_caster_is_used(self):
- self.cur.execute("select ?", (4,))
- val = self.cur.fetchone()[0]
- self.assertEqual(type(val), float)
-
-@unittest.skipUnless(zlib, "requires zlib")
-class BinaryConverterTests(unittest.TestCase):
- def convert(s):
- return zlib.decompress(s)
- convert = staticmethod(convert)
-
- def setUp(self):
- self.con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES)
- sqlite.register_converter("bin", BinaryConverterTests.convert)
-
- def tearDown(self):
- self.con.close()
-
- def test_binary_input_for_converter(self):
- testdata = b"abcdefg" * 10
- result = self.con.execute('select ? as "x [bin]"', (memoryview(zlib.compress(testdata)),)).fetchone()[0]
- self.assertEqual(testdata, result)
-
-class DateTimeTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES)
- self.cur = self.con.cursor()
- self.cur.execute("create table test(d date, ts timestamp)")
-
- def tearDown(self):
- self.cur.close()
- self.con.close()
-
- def test_sqlite_date(self):
- d = sqlite.Date(2004, 2, 14)
- self.cur.execute("insert into test(d) values (?)", (d,))
- self.cur.execute("select d from test")
- d2 = self.cur.fetchone()[0]
- self.assertEqual(d, d2)
-
- def test_sqlite_timestamp(self):
- ts = sqlite.Timestamp(2004, 2, 14, 7, 15, 0)
- self.cur.execute("insert into test(ts) values (?)", (ts,))
- self.cur.execute("select ts from test")
- ts2 = self.cur.fetchone()[0]
- self.assertEqual(ts, ts2)
-
- def test_sql_timestamp(self):
- now = datetime.datetime.utcnow()
- self.cur.execute("insert into test(ts) values (current_timestamp)")
- self.cur.execute("select ts from test")
- ts = self.cur.fetchone()[0]
- self.assertEqual(type(ts), datetime.datetime)
- self.assertEqual(ts.year, now.year)
-
- def test_date_time_sub_seconds(self):
- ts = sqlite.Timestamp(2004, 2, 14, 7, 15, 0, 500000)
- self.cur.execute("insert into test(ts) values (?)", (ts,))
- self.cur.execute("select ts from test")
- ts2 = self.cur.fetchone()[0]
- self.assertEqual(ts, ts2)
-
- def test_date_time_sub_seconds_floating_point(self):
- ts = sqlite.Timestamp(2004, 2, 14, 7, 15, 0, 510241)
- self.cur.execute("insert into test(ts) values (?)", (ts,))
- self.cur.execute("select ts from test")
- ts2 = self.cur.fetchone()[0]
- self.assertEqual(ts, ts2)
-
-def suite():
- tests = [
- BinaryConverterTests,
- ColNamesTests,
- CommonTableExpressionTests,
- DateTimeTests,
- DeclTypesTests,
- ObjectAdaptationTests,
- SqliteTypeTests,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sqlite3/test/userfunctions.py b/common/py3-stdlib/sqlite3/test/userfunctions.py
deleted file mode 100644
index 539fd4b..0000000
--- a/common/py3-stdlib/sqlite3/test/userfunctions.py
+++ /dev/null
@@ -1,572 +0,0 @@
-# pysqlite2/test/userfunctions.py: tests for user-defined functions and
-# aggregates.
-#
-# Copyright (C) 2005-2007 Gerhard Häring <gh@ghaering.de>
-#
-# This file is part of pysqlite.
-#
-# This software is provided 'as-is', without any express or implied
-# warranty. In no event will the authors be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-# claim that you wrote the original software. If you use this software
-# in a product, an acknowledgment in the product documentation would be
-# appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-# misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-
-import unittest
-import unittest.mock
-import sqlite3 as sqlite
-
-from test.support import gc_collect
-
-
-def func_returntext():
- return "foo"
-def func_returntextwithnull():
- return "1\x002"
-def func_returnunicode():
- return "bar"
-def func_returnint():
- return 42
-def func_returnfloat():
- return 3.14
-def func_returnnull():
- return None
-def func_returnblob():
- return b"blob"
-def func_returnlonglong():
- return 1<<31
-def func_raiseexception():
- 5/0
-
-class AggrNoStep:
- def __init__(self):
- pass
-
- def finalize(self):
- return 1
-
-class AggrNoFinalize:
- def __init__(self):
- pass
-
- def step(self, x):
- pass
-
-class AggrExceptionInInit:
- def __init__(self):
- 5/0
-
- def step(self, x):
- pass
-
- def finalize(self):
- pass
-
-class AggrExceptionInStep:
- def __init__(self):
- pass
-
- def step(self, x):
- 5/0
-
- def finalize(self):
- return 42
-
-class AggrExceptionInFinalize:
- def __init__(self):
- pass
-
- def step(self, x):
- pass
-
- def finalize(self):
- 5/0
-
-class AggrCheckType:
- def __init__(self):
- self.val = None
-
- def step(self, whichType, val):
- theType = {"str": str, "int": int, "float": float, "None": type(None),
- "blob": bytes}
- self.val = int(theType[whichType] is type(val))
-
- def finalize(self):
- return self.val
-
-class AggrCheckTypes:
- def __init__(self):
- self.val = 0
-
- def step(self, whichType, *vals):
- theType = {"str": str, "int": int, "float": float, "None": type(None),
- "blob": bytes}
- for val in vals:
- self.val += int(theType[whichType] is type(val))
-
- def finalize(self):
- return self.val
-
-class AggrSum:
- def __init__(self):
- self.val = 0.0
-
- def step(self, val):
- self.val += val
-
- def finalize(self):
- return self.val
-
-class AggrText:
- def __init__(self):
- self.txt = ""
- def step(self, txt):
- self.txt = self.txt + txt
- def finalize(self):
- return self.txt
-
-
-class FunctionTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
-
- self.con.create_function("returntext", 0, func_returntext)
- self.con.create_function("returntextwithnull", 0, func_returntextwithnull)
- self.con.create_function("returnunicode", 0, func_returnunicode)
- self.con.create_function("returnint", 0, func_returnint)
- self.con.create_function("returnfloat", 0, func_returnfloat)
- self.con.create_function("returnnull", 0, func_returnnull)
- self.con.create_function("returnblob", 0, func_returnblob)
- self.con.create_function("returnlonglong", 0, func_returnlonglong)
- self.con.create_function("returnnan", 0, lambda: float("nan"))
- self.con.create_function("returntoolargeint", 0, lambda: 1 << 65)
- self.con.create_function("raiseexception", 0, func_raiseexception)
-
- self.con.create_function("isblob", 1, lambda x: isinstance(x, bytes))
- self.con.create_function("isnone", 1, lambda x: x is None)
- self.con.create_function("spam", -1, lambda *x: len(x))
- self.con.execute("create table test(t text)")
-
- def tearDown(self):
- self.con.close()
-
- def test_func_error_on_create(self):
- with self.assertRaises(sqlite.OperationalError):
- self.con.create_function("bla", -100, lambda x: 2*x)
-
- def test_func_ref_count(self):
- def getfunc():
- def f():
- return 1
- return f
- f = getfunc()
- globals()["foo"] = f
- # self.con.create_function("reftest", 0, getfunc())
- self.con.create_function("reftest", 0, f)
- cur = self.con.cursor()
- cur.execute("select reftest()")
-
- def test_func_return_text(self):
- cur = self.con.cursor()
- cur.execute("select returntext()")
- val = cur.fetchone()[0]
- self.assertEqual(type(val), str)
- self.assertEqual(val, "foo")
-
- def test_func_return_text_with_null_char(self):
- cur = self.con.cursor()
- res = cur.execute("select returntextwithnull()").fetchone()[0]
- self.assertEqual(type(res), str)
- self.assertEqual(res, "1\x002")
-
- def test_func_return_unicode(self):
- cur = self.con.cursor()
- cur.execute("select returnunicode()")
- val = cur.fetchone()[0]
- self.assertEqual(type(val), str)
- self.assertEqual(val, "bar")
-
- def test_func_return_int(self):
- cur = self.con.cursor()
- cur.execute("select returnint()")
- val = cur.fetchone()[0]
- self.assertEqual(type(val), int)
- self.assertEqual(val, 42)
-
- def test_func_return_float(self):
- cur = self.con.cursor()
- cur.execute("select returnfloat()")
- val = cur.fetchone()[0]
- self.assertEqual(type(val), float)
- if val < 3.139 or val > 3.141:
- self.fail("wrong value")
-
- def test_func_return_null(self):
- cur = self.con.cursor()
- cur.execute("select returnnull()")
- val = cur.fetchone()[0]
- self.assertEqual(type(val), type(None))
- self.assertEqual(val, None)
-
- def test_func_return_blob(self):
- cur = self.con.cursor()
- cur.execute("select returnblob()")
- val = cur.fetchone()[0]
- self.assertEqual(type(val), bytes)
- self.assertEqual(val, b"blob")
-
- def test_func_return_long_long(self):
- cur = self.con.cursor()
- cur.execute("select returnlonglong()")
- val = cur.fetchone()[0]
- self.assertEqual(val, 1<<31)
-
- def test_func_return_nan(self):
- cur = self.con.cursor()
- cur.execute("select returnnan()")
- self.assertIsNone(cur.fetchone()[0])
-
- def test_func_return_too_large_int(self):
- cur = self.con.cursor()
- with self.assertRaises(sqlite.OperationalError):
- self.con.execute("select returntoolargeint()")
-
- def test_func_exception(self):
- cur = self.con.cursor()
- with self.assertRaises(sqlite.OperationalError) as cm:
- cur.execute("select raiseexception()")
- cur.fetchone()
- self.assertEqual(str(cm.exception), 'user-defined function raised exception')
-
- def test_any_arguments(self):
- cur = self.con.cursor()
- cur.execute("select spam(?, ?)", (1, 2))
- val = cur.fetchone()[0]
- self.assertEqual(val, 2)
-
- def test_empty_blob(self):
- cur = self.con.execute("select isblob(x'')")
- self.assertTrue(cur.fetchone()[0])
-
- def test_nan_float(self):
- cur = self.con.execute("select isnone(?)", (float("nan"),))
- # SQLite has no concept of nan; it is converted to NULL
- self.assertTrue(cur.fetchone()[0])
-
- def test_too_large_int(self):
- err = "Python int too large to convert to SQLite INTEGER"
- self.assertRaisesRegex(OverflowError, err, self.con.execute,
- "select spam(?)", (1 << 65,))
-
- def test_non_contiguous_blob(self):
- self.assertRaisesRegex(ValueError, "could not convert BLOB to buffer",
- self.con.execute, "select spam(?)",
- (memoryview(b"blob")[::2],))
-
- def test_param_surrogates(self):
- self.assertRaisesRegex(UnicodeEncodeError, "surrogates not allowed",
- self.con.execute, "select spam(?)",
- ("\ud803\ude6d",))
-
- def test_func_params(self):
- results = []
- def append_result(arg):
- results.append((arg, type(arg)))
- self.con.create_function("test_params", 1, append_result)
-
- dataset = [
- (42, int),
- (-1, int),
- (1234567890123456789, int),
- (4611686018427387905, int), # 63-bit int with non-zero low bits
- (3.14, float),
- (float('inf'), float),
- ("text", str),
- ("1\x002", str),
- ("\u02e2q\u02e1\u2071\u1d57\u1d49", str),
- (b"blob", bytes),
- (bytearray(range(2)), bytes),
- (memoryview(b"blob"), bytes),
- (None, type(None)),
- ]
- for val, _ in dataset:
- cur = self.con.execute("select test_params(?)", (val,))
- cur.fetchone()
- self.assertEqual(dataset, results)
-
- # Regarding deterministic functions:
- #
- # Between 3.8.3 and 3.15.0, deterministic functions were only used to
- # optimize inner loops, so for those versions we can only test if the
- # sqlite machinery has factored out a call or not. From 3.15.0 and onward,
- # deterministic functions were permitted in WHERE clauses of partial
- # indices, which allows testing based on syntax, iso. the query optimizer.
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher")
- def test_func_non_deterministic(self):
- mock = unittest.mock.Mock(return_value=None)
- self.con.create_function("nondeterministic", 0, mock, deterministic=False)
- if sqlite.sqlite_version_info < (3, 15, 0):
- self.con.execute("select nondeterministic() = nondeterministic()")
- self.assertEqual(mock.call_count, 2)
- else:
- with self.assertRaises(sqlite.OperationalError):
- self.con.execute("create index t on test(t) where nondeterministic() is not null")
-
- @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher")
- def test_func_deterministic(self):
- mock = unittest.mock.Mock(return_value=None)
- self.con.create_function("deterministic", 0, mock, deterministic=True)
- if sqlite.sqlite_version_info < (3, 15, 0):
- self.con.execute("select deterministic() = deterministic()")
- self.assertEqual(mock.call_count, 1)
- else:
- try:
- self.con.execute("create index t on test(t) where deterministic() is not null")
- except sqlite.OperationalError:
- self.fail("Unexpected failure while creating partial index")
-
- @unittest.skipIf(sqlite.sqlite_version_info >= (3, 8, 3), "SQLite < 3.8.3 needed")
- def test_func_deterministic_not_supported(self):
- with self.assertRaises(sqlite.NotSupportedError):
- self.con.create_function("deterministic", 0, int, deterministic=True)
-
- def test_func_deterministic_keyword_only(self):
- with self.assertRaises(TypeError):
- self.con.create_function("deterministic", 0, int, True)
-
- def test_function_destructor_via_gc(self):
- # See bpo-44304: The destructor of the user function can
- # crash if is called without the GIL from the gc functions
- dest = sqlite.connect(':memory:')
- def md5sum(t):
- return
-
- dest.create_function("md5", 1, md5sum)
- x = dest("create table lang (name, first_appeared)")
- del md5sum, dest
-
- y = [x]
- y.append(y)
-
- del x,y
- gc_collect()
-
-class AggregateTests(unittest.TestCase):
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- cur = self.con.cursor()
- cur.execute("""
- create table test(
- t text,
- i integer,
- f float,
- n,
- b blob
- )
- """)
- cur.execute("insert into test(t, i, f, n, b) values (?, ?, ?, ?, ?)",
- ("foo", 5, 3.14, None, memoryview(b"blob"),))
-
- self.con.create_aggregate("nostep", 1, AggrNoStep)
- self.con.create_aggregate("nofinalize", 1, AggrNoFinalize)
- self.con.create_aggregate("excInit", 1, AggrExceptionInInit)
- self.con.create_aggregate("excStep", 1, AggrExceptionInStep)
- self.con.create_aggregate("excFinalize", 1, AggrExceptionInFinalize)
- self.con.create_aggregate("checkType", 2, AggrCheckType)
- self.con.create_aggregate("checkTypes", -1, AggrCheckTypes)
- self.con.create_aggregate("mysum", 1, AggrSum)
- self.con.create_aggregate("aggtxt", 1, AggrText)
-
- def tearDown(self):
- #self.cur.close()
- #self.con.close()
- pass
-
- def test_aggr_error_on_create(self):
- with self.assertRaises(sqlite.OperationalError):
- self.con.create_function("bla", -100, AggrSum)
-
- def test_aggr_no_step(self):
- cur = self.con.cursor()
- with self.assertRaises(AttributeError) as cm:
- cur.execute("select nostep(t) from test")
- self.assertEqual(str(cm.exception), "'AggrNoStep' object has no attribute 'step'")
-
- def test_aggr_no_finalize(self):
- cur = self.con.cursor()
- with self.assertRaises(sqlite.OperationalError) as cm:
- cur.execute("select nofinalize(t) from test")
- val = cur.fetchone()[0]
- self.assertEqual(str(cm.exception), "user-defined aggregate's 'finalize' method raised error")
-
- def test_aggr_exception_in_init(self):
- cur = self.con.cursor()
- with self.assertRaises(sqlite.OperationalError) as cm:
- cur.execute("select excInit(t) from test")
- val = cur.fetchone()[0]
- self.assertEqual(str(cm.exception), "user-defined aggregate's '__init__' method raised error")
-
- def test_aggr_exception_in_step(self):
- cur = self.con.cursor()
- with self.assertRaises(sqlite.OperationalError) as cm:
- cur.execute("select excStep(t) from test")
- val = cur.fetchone()[0]
- self.assertEqual(str(cm.exception), "user-defined aggregate's 'step' method raised error")
-
- def test_aggr_exception_in_finalize(self):
- cur = self.con.cursor()
- with self.assertRaises(sqlite.OperationalError) as cm:
- cur.execute("select excFinalize(t) from test")
- val = cur.fetchone()[0]
- self.assertEqual(str(cm.exception), "user-defined aggregate's 'finalize' method raised error")
-
- def test_aggr_check_param_str(self):
- cur = self.con.cursor()
- cur.execute("select checkTypes('str', ?, ?)", ("foo", str()))
- val = cur.fetchone()[0]
- self.assertEqual(val, 2)
-
- def test_aggr_check_param_int(self):
- cur = self.con.cursor()
- cur.execute("select checkType('int', ?)", (42,))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def test_aggr_check_params_int(self):
- cur = self.con.cursor()
- cur.execute("select checkTypes('int', ?, ?)", (42, 24))
- val = cur.fetchone()[0]
- self.assertEqual(val, 2)
-
- def test_aggr_check_param_float(self):
- cur = self.con.cursor()
- cur.execute("select checkType('float', ?)", (3.14,))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def test_aggr_check_param_none(self):
- cur = self.con.cursor()
- cur.execute("select checkType('None', ?)", (None,))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def test_aggr_check_param_blob(self):
- cur = self.con.cursor()
- cur.execute("select checkType('blob', ?)", (memoryview(b"blob"),))
- val = cur.fetchone()[0]
- self.assertEqual(val, 1)
-
- def test_aggr_check_aggr_sum(self):
- cur = self.con.cursor()
- cur.execute("delete from test")
- cur.executemany("insert into test(i) values (?)", [(10,), (20,), (30,)])
- cur.execute("select mysum(i) from test")
- val = cur.fetchone()[0]
- self.assertEqual(val, 60)
-
- def test_aggr_no_match(self):
- cur = self.con.execute("select mysum(i) from (select 1 as i) where i == 0")
- val = cur.fetchone()[0]
- self.assertIsNone(val)
-
- def test_aggr_text(self):
- cur = self.con.cursor()
- for txt in ["foo", "1\x002"]:
- with self.subTest(txt=txt):
- cur.execute("select aggtxt(?) from test", (txt,))
- val = cur.fetchone()[0]
- self.assertEqual(val, txt)
-
-
-class AuthorizerTests(unittest.TestCase):
- @staticmethod
- def authorizer_cb(action, arg1, arg2, dbname, source):
- if action != sqlite.SQLITE_SELECT:
- return sqlite.SQLITE_DENY
- if arg2 == 'c2' or arg1 == 't2':
- return sqlite.SQLITE_DENY
- return sqlite.SQLITE_OK
-
- def setUp(self):
- self.con = sqlite.connect(":memory:")
- self.con.executescript("""
- create table t1 (c1, c2);
- create table t2 (c1, c2);
- insert into t1 (c1, c2) values (1, 2);
- insert into t2 (c1, c2) values (4, 5);
- """)
-
- # For our security test:
- self.con.execute("select c2 from t2")
-
- self.con.set_authorizer(self.authorizer_cb)
-
- def tearDown(self):
- pass
-
- def test_table_access(self):
- with self.assertRaises(sqlite.DatabaseError) as cm:
- self.con.execute("select * from t2")
- self.assertIn('prohibited', str(cm.exception))
-
- def test_column_access(self):
- with self.assertRaises(sqlite.DatabaseError) as cm:
- self.con.execute("select c2 from t1")
- self.assertIn('prohibited', str(cm.exception))
-
-class AuthorizerRaiseExceptionTests(AuthorizerTests):
- @staticmethod
- def authorizer_cb(action, arg1, arg2, dbname, source):
- if action != sqlite.SQLITE_SELECT:
- raise ValueError
- if arg2 == 'c2' or arg1 == 't2':
- raise ValueError
- return sqlite.SQLITE_OK
-
-class AuthorizerIllegalTypeTests(AuthorizerTests):
- @staticmethod
- def authorizer_cb(action, arg1, arg2, dbname, source):
- if action != sqlite.SQLITE_SELECT:
- return 0.0
- if arg2 == 'c2' or arg1 == 't2':
- return 0.0
- return sqlite.SQLITE_OK
-
-class AuthorizerLargeIntegerTests(AuthorizerTests):
- @staticmethod
- def authorizer_cb(action, arg1, arg2, dbname, source):
- if action != sqlite.SQLITE_SELECT:
- return 2**32
- if arg2 == 'c2' or arg1 == 't2':
- return 2**32
- return sqlite.SQLITE_OK
-
-
-def suite():
- tests = [
- AggregateTests,
- AuthorizerIllegalTypeTests,
- AuthorizerLargeIntegerTests,
- AuthorizerRaiseExceptionTests,
- AuthorizerTests,
- FunctionTests,
- ]
- return unittest.TestSuite(
- [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests]
- )
-
-def test():
- runner = unittest.TextTestRunner()
- runner.run(suite())
-
-if __name__ == "__main__":
- test()
diff --git a/common/py3-stdlib/sre_compile.py b/common/py3-stdlib/sre_compile.py
index aed752d..f9da61e 100644
--- a/common/py3-stdlib/sre_compile.py
+++ b/common/py3-stdlib/sre_compile.py
@@ -1,808 +1,7 @@
-#
-# Secret Labs' Regular Expression Engine
-#
-# convert template to internal format
-#
-# Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved.
-#
-# See the sre.py file for information on usage and redistribution.
-#
+import warnings
+warnings.warn(f"module {__name__!r} is deprecated",
+ DeprecationWarning,
+ stacklevel=2)
-"""Internal support module for sre"""
-
-import _sre
-import sre_parse
-from sre_constants import *
-
-assert _sre.MAGIC == MAGIC, "SRE module mismatch"
-
-_LITERAL_CODES = {LITERAL, NOT_LITERAL}
-_REPEATING_CODES = {REPEAT, MIN_REPEAT, MAX_REPEAT}
-_SUCCESS_CODES = {SUCCESS, FAILURE}
-_ASSERT_CODES = {ASSERT, ASSERT_NOT}
-_UNIT_CODES = _LITERAL_CODES | {ANY, IN}
-
-# Sets of lowercase characters which have the same uppercase.
-_equivalences = (
- # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I
- (0x69, 0x131), # iı
- # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S
- (0x73, 0x17f), # sſ
- # MICRO SIGN, GREEK SMALL LETTER MU
- (0xb5, 0x3bc), # µμ
- # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
- (0x345, 0x3b9, 0x1fbe), # \u0345ιι
- # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
- (0x390, 0x1fd3), # ΐΐ
- # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
- (0x3b0, 0x1fe3), # ΰΰ
- # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL
- (0x3b2, 0x3d0), # βϐ
- # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL
- (0x3b5, 0x3f5), # εϵ
- # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL
- (0x3b8, 0x3d1), # θϑ
- # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL
- (0x3ba, 0x3f0), # κϰ
- # GREEK SMALL LETTER PI, GREEK PI SYMBOL
- (0x3c0, 0x3d6), # πϖ
- # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL
- (0x3c1, 0x3f1), # ρϱ
- # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA
- (0x3c2, 0x3c3), # ςσ
- # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL
- (0x3c6, 0x3d5), # φϕ
- # CYRILLIC SMALL LETTER VE, CYRILLIC SMALL LETTER ROUNDED VE
- (0x432, 0x1c80), # вᲀ
- # CYRILLIC SMALL LETTER DE, CYRILLIC SMALL LETTER LONG-LEGGED DE
- (0x434, 0x1c81), # дᲁ
- # CYRILLIC SMALL LETTER O, CYRILLIC SMALL LETTER NARROW O
- (0x43e, 0x1c82), # оᲂ
- # CYRILLIC SMALL LETTER ES, CYRILLIC SMALL LETTER WIDE ES
- (0x441, 0x1c83), # сᲃ
- # CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
- (0x442, 0x1c84, 0x1c85), # тᲄᲅ
- # CYRILLIC SMALL LETTER HARD SIGN, CYRILLIC SMALL LETTER TALL HARD SIGN
- (0x44a, 0x1c86), # ъᲆ
- # CYRILLIC SMALL LETTER YAT, CYRILLIC SMALL LETTER TALL YAT
- (0x463, 0x1c87), # ѣᲇ
- # CYRILLIC SMALL LETTER UNBLENDED UK, CYRILLIC SMALL LETTER MONOGRAPH UK
- (0x1c88, 0xa64b), # ᲈꙋ
- # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE
- (0x1e61, 0x1e9b), # ṡẛ
- # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST
- (0xfb05, 0xfb06), # ſtst
-)
-
-# Maps the lowercase code to lowercase codes which have the same uppercase.
-_ignorecase_fixes = {i: tuple(j for j in t if i != j)
- for t in _equivalences for i in t}
-
-def _combine_flags(flags, add_flags, del_flags,
- TYPE_FLAGS=sre_parse.TYPE_FLAGS):
- if add_flags & TYPE_FLAGS:
- flags &= ~TYPE_FLAGS
- return (flags | add_flags) & ~del_flags
-
-def _compile(code, pattern, flags):
- # internal: compile a (sub)pattern
- emit = code.append
- _len = len
- LITERAL_CODES = _LITERAL_CODES
- REPEATING_CODES = _REPEATING_CODES
- SUCCESS_CODES = _SUCCESS_CODES
- ASSERT_CODES = _ASSERT_CODES
- iscased = None
- tolower = None
- fixes = None
- if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE:
- if flags & SRE_FLAG_UNICODE:
- iscased = _sre.unicode_iscased
- tolower = _sre.unicode_tolower
- fixes = _ignorecase_fixes
- else:
- iscased = _sre.ascii_iscased
- tolower = _sre.ascii_tolower
- for op, av in pattern:
- if op in LITERAL_CODES:
- if not flags & SRE_FLAG_IGNORECASE:
- emit(op)
- emit(av)
- elif flags & SRE_FLAG_LOCALE:
- emit(OP_LOCALE_IGNORE[op])
- emit(av)
- elif not iscased(av):
- emit(op)
- emit(av)
- else:
- lo = tolower(av)
- if not fixes: # ascii
- emit(OP_IGNORE[op])
- emit(lo)
- elif lo not in fixes:
- emit(OP_UNICODE_IGNORE[op])
- emit(lo)
- else:
- emit(IN_UNI_IGNORE)
- skip = _len(code); emit(0)
- if op is NOT_LITERAL:
- emit(NEGATE)
- for k in (lo,) + fixes[lo]:
- emit(LITERAL)
- emit(k)
- emit(FAILURE)
- code[skip] = _len(code) - skip
- elif op is IN:
- charset, hascased = _optimize_charset(av, iscased, tolower, fixes)
- if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE:
- emit(IN_LOC_IGNORE)
- elif not hascased:
- emit(IN)
- elif not fixes: # ascii
- emit(IN_IGNORE)
- else:
- emit(IN_UNI_IGNORE)
- skip = _len(code); emit(0)
- _compile_charset(charset, flags, code)
- code[skip] = _len(code) - skip
- elif op is ANY:
- if flags & SRE_FLAG_DOTALL:
- emit(ANY_ALL)
- else:
- emit(ANY)
- elif op in REPEATING_CODES:
- if flags & SRE_FLAG_TEMPLATE:
- raise error("internal: unsupported template operator %r" % (op,))
- if _simple(av[2]):
- if op is MAX_REPEAT:
- emit(REPEAT_ONE)
- else:
- emit(MIN_REPEAT_ONE)
- skip = _len(code); emit(0)
- emit(av[0])
- emit(av[1])
- _compile(code, av[2], flags)
- emit(SUCCESS)
- code[skip] = _len(code) - skip
- else:
- emit(REPEAT)
- skip = _len(code); emit(0)
- emit(av[0])
- emit(av[1])
- _compile(code, av[2], flags)
- code[skip] = _len(code) - skip
- if op is MAX_REPEAT:
- emit(MAX_UNTIL)
- else:
- emit(MIN_UNTIL)
- elif op is SUBPATTERN:
- group, add_flags, del_flags, p = av
- if group:
- emit(MARK)
- emit((group-1)*2)
- # _compile_info(code, p, _combine_flags(flags, add_flags, del_flags))
- _compile(code, p, _combine_flags(flags, add_flags, del_flags))
- if group:
- emit(MARK)
- emit((group-1)*2+1)
- elif op in SUCCESS_CODES:
- emit(op)
- elif op in ASSERT_CODES:
- emit(op)
- skip = _len(code); emit(0)
- if av[0] >= 0:
- emit(0) # look ahead
- else:
- lo, hi = av[1].getwidth()
- if lo != hi:
- raise error("look-behind requires fixed-width pattern")
- emit(lo) # look behind
- _compile(code, av[1], flags)
- emit(SUCCESS)
- code[skip] = _len(code) - skip
- elif op is CALL:
- emit(op)
- skip = _len(code); emit(0)
- _compile(code, av, flags)
- emit(SUCCESS)
- code[skip] = _len(code) - skip
- elif op is AT:
- emit(op)
- if flags & SRE_FLAG_MULTILINE:
- av = AT_MULTILINE.get(av, av)
- if flags & SRE_FLAG_LOCALE:
- av = AT_LOCALE.get(av, av)
- elif flags & SRE_FLAG_UNICODE:
- av = AT_UNICODE.get(av, av)
- emit(av)
- elif op is BRANCH:
- emit(op)
- tail = []
- tailappend = tail.append
- for av in av[1]:
- skip = _len(code); emit(0)
- # _compile_info(code, av, flags)
- _compile(code, av, flags)
- emit(JUMP)
- tailappend(_len(code)); emit(0)
- code[skip] = _len(code) - skip
- emit(FAILURE) # end of branch
- for tail in tail:
- code[tail] = _len(code) - tail
- elif op is CATEGORY:
- emit(op)
- if flags & SRE_FLAG_LOCALE:
- av = CH_LOCALE[av]
- elif flags & SRE_FLAG_UNICODE:
- av = CH_UNICODE[av]
- emit(av)
- elif op is GROUPREF:
- if not flags & SRE_FLAG_IGNORECASE:
- emit(op)
- elif flags & SRE_FLAG_LOCALE:
- emit(GROUPREF_LOC_IGNORE)
- elif not fixes: # ascii
- emit(GROUPREF_IGNORE)
- else:
- emit(GROUPREF_UNI_IGNORE)
- emit(av-1)
- elif op is GROUPREF_EXISTS:
- emit(op)
- emit(av[0]-1)
- skipyes = _len(code); emit(0)
- _compile(code, av[1], flags)
- if av[2]:
- emit(JUMP)
- skipno = _len(code); emit(0)
- code[skipyes] = _len(code) - skipyes + 1
- _compile(code, av[2], flags)
- code[skipno] = _len(code) - skipno
- else:
- code[skipyes] = _len(code) - skipyes + 1
- else:
- raise error("internal: unsupported operand type %r" % (op,))
-
-def _compile_charset(charset, flags, code):
- # compile charset subprogram
- emit = code.append
- for op, av in charset:
- emit(op)
- if op is NEGATE:
- pass
- elif op is LITERAL:
- emit(av)
- elif op is RANGE or op is RANGE_UNI_IGNORE:
- emit(av[0])
- emit(av[1])
- elif op is CHARSET:
- code.extend(av)
- elif op is BIGCHARSET:
- code.extend(av)
- elif op is CATEGORY:
- if flags & SRE_FLAG_LOCALE:
- emit(CH_LOCALE[av])
- elif flags & SRE_FLAG_UNICODE:
- emit(CH_UNICODE[av])
- else:
- emit(av)
- else:
- raise error("internal: unsupported set operator %r" % (op,))
- emit(FAILURE)
-
-def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
- # internal: optimize character set
- out = []
- tail = []
- charmap = bytearray(256)
- hascased = False
- for op, av in charset:
- while True:
- try:
- if op is LITERAL:
- if fixup:
- lo = fixup(av)
- charmap[lo] = 1
- if fixes and lo in fixes:
- for k in fixes[lo]:
- charmap[k] = 1
- if not hascased and iscased(av):
- hascased = True
- else:
- charmap[av] = 1
- elif op is RANGE:
- r = range(av[0], av[1]+1)
- if fixup:
- if fixes:
- for i in map(fixup, r):
- charmap[i] = 1
- if i in fixes:
- for k in fixes[i]:
- charmap[k] = 1
- else:
- for i in map(fixup, r):
- charmap[i] = 1
- if not hascased:
- hascased = any(map(iscased, r))
- else:
- for i in r:
- charmap[i] = 1
- elif op is NEGATE:
- out.append((op, av))
- else:
- tail.append((op, av))
- except IndexError:
- if len(charmap) == 256:
- # character set contains non-UCS1 character codes
- charmap += b'\0' * 0xff00
- continue
- # Character set contains non-BMP character codes.
- # For range, all BMP characters in the range are already
- # proceeded.
- if fixup:
- hascased = True
- # For now, IN_UNI_IGNORE+LITERAL and
- # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP
- # characters, because two characters (at least one of
- # which is not in the BMP) match case-insensitively
- # if and only if:
- # 1) c1.lower() == c2.lower()
- # 2) c1.lower() == c2 or c1.lower().upper() == c2
- # Also, both c.lower() and c.lower().upper() are single
- # characters for every non-BMP character.
- if op is RANGE:
- op = RANGE_UNI_IGNORE
- tail.append((op, av))
- break
-
- # compress character map
- runs = []
- q = 0
- while True:
- p = charmap.find(1, q)
- if p < 0:
- break
- if len(runs) >= 2:
- runs = None
- break
- q = charmap.find(0, p)
- if q < 0:
- runs.append((p, len(charmap)))
- break
- runs.append((p, q))
- if runs is not None:
- # use literal/range
- for p, q in runs:
- if q - p == 1:
- out.append((LITERAL, p))
- else:
- out.append((RANGE, (p, q - 1)))
- out += tail
- # if the case was changed or new representation is more compact
- if hascased or len(out) < len(charset):
- return out, hascased
- # else original character set is good enough
- return charset, hascased
-
- # use bitmap
- if len(charmap) == 256:
- data = _mk_bitmap(charmap)
- out.append((CHARSET, data))
- out += tail
- return out, hascased
-
- # To represent a big charset, first a bitmap of all characters in the
- # set is constructed. Then, this bitmap is sliced into chunks of 256
- # characters, duplicate chunks are eliminated, and each chunk is
- # given a number. In the compiled expression, the charset is
- # represented by a 32-bit word sequence, consisting of one word for
- # the number of different chunks, a sequence of 256 bytes (64 words)
- # of chunk numbers indexed by their original chunk position, and a
- # sequence of 256-bit chunks (8 words each).
-
- # Compression is normally good: in a typical charset, large ranges of
- # Unicode will be either completely excluded (e.g. if only cyrillic
- # letters are to be matched), or completely included (e.g. if large
- # subranges of Kanji match). These ranges will be represented by
- # chunks of all one-bits or all zero-bits.
-
- # Matching can be also done efficiently: the more significant byte of
- # the Unicode character is an index into the chunk number, and the
- # less significant byte is a bit index in the chunk (just like the
- # CHARSET matching).
-
- charmap = bytes(charmap) # should be hashable
- comps = {}
- mapping = bytearray(256)
- block = 0
- data = bytearray()
- for i in range(0, 65536, 256):
- chunk = charmap[i: i + 256]
- if chunk in comps:
- mapping[i // 256] = comps[chunk]
- else:
- mapping[i // 256] = comps[chunk] = block
- block += 1
- data += chunk
- data = _mk_bitmap(data)
- data[0:0] = [block] + _bytes_to_codes(mapping)
- out.append((BIGCHARSET, data))
- out += tail
- return out, hascased
-
-_CODEBITS = _sre.CODESIZE * 8
-MAXCODE = (1 << _CODEBITS) - 1
-_BITS_TRANS = b'0' + b'1' * 255
-def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int):
- s = bits.translate(_BITS_TRANS)[::-1]
- return [_int(s[i - _CODEBITS: i], 2)
- for i in range(len(s), 0, -_CODEBITS)]
-
-def _bytes_to_codes(b):
- # Convert block indices to word array
- a = memoryview(b).cast('I')
- assert a.itemsize == _sre.CODESIZE
- assert len(a) * a.itemsize == len(b)
- return a.tolist()
-
-def _simple(p):
- # check if this subpattern is a "simple" operator
- if len(p) != 1:
- return False
- op, av = p[0]
- if op is SUBPATTERN:
- return av[0] is None and _simple(av[-1])
- return op in _UNIT_CODES
-
-def _generate_overlap_table(prefix):
- """
- Generate an overlap table for the following prefix.
- An overlap table is a table of the same size as the prefix which
- informs about the potential self-overlap for each index in the prefix:
- - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...]
- - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with
- prefix[0:k]
- """
- table = [0] * len(prefix)
- for i in range(1, len(prefix)):
- idx = table[i - 1]
- while prefix[i] != prefix[idx]:
- if idx == 0:
- table[i] = 0
- break
- idx = table[idx - 1]
- else:
- table[i] = idx + 1
- return table
-
-def _get_iscased(flags):
- if not flags & SRE_FLAG_IGNORECASE:
- return None
- elif flags & SRE_FLAG_UNICODE:
- return _sre.unicode_iscased
- else:
- return _sre.ascii_iscased
-
-def _get_literal_prefix(pattern, flags):
- # look for literal prefix
- prefix = []
- prefixappend = prefix.append
- prefix_skip = None
- iscased = _get_iscased(flags)
- for op, av in pattern.data:
- if op is LITERAL:
- if iscased and iscased(av):
- break
- prefixappend(av)
- elif op is SUBPATTERN:
- group, add_flags, del_flags, p = av
- flags1 = _combine_flags(flags, add_flags, del_flags)
- if flags1 & SRE_FLAG_IGNORECASE and flags1 & SRE_FLAG_LOCALE:
- break
- prefix1, prefix_skip1, got_all = _get_literal_prefix(p, flags1)
- if prefix_skip is None:
- if group is not None:
- prefix_skip = len(prefix)
- elif prefix_skip1 is not None:
- prefix_skip = len(prefix) + prefix_skip1
- prefix.extend(prefix1)
- if not got_all:
- break
- else:
- break
- else:
- return prefix, prefix_skip, True
- return prefix, prefix_skip, False
-
-def _get_charset_prefix(pattern, flags):
- while True:
- if not pattern.data:
- return None
- op, av = pattern.data[0]
- if op is not SUBPATTERN:
- break
- group, add_flags, del_flags, pattern = av
- flags = _combine_flags(flags, add_flags, del_flags)
- if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE:
- return None
-
- iscased = _get_iscased(flags)
- if op is LITERAL:
- if iscased and iscased(av):
- return None
- return [(op, av)]
- elif op is BRANCH:
- charset = []
- charsetappend = charset.append
- for p in av[1]:
- if not p:
- return None
- op, av = p[0]
- if op is LITERAL and not (iscased and iscased(av)):
- charsetappend((op, av))
- else:
- return None
- return charset
- elif op is IN:
- charset = av
- if iscased:
- for op, av in charset:
- if op is LITERAL:
- if iscased(av):
- return None
- elif op is RANGE:
- if av[1] > 0xffff:
- return None
- if any(map(iscased, range(av[0], av[1]+1))):
- return None
- return charset
- return None
-
-def _compile_info(code, pattern, flags):
- # internal: compile an info block. in the current version,
- # this contains min/max pattern width, and an optional literal
- # prefix or a character map
- lo, hi = pattern.getwidth()
- if hi > MAXCODE:
- hi = MAXCODE
- if lo == 0:
- code.extend([INFO, 4, 0, lo, hi])
- return
- # look for a literal prefix
- prefix = []
- prefix_skip = 0
- charset = [] # not used
- if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE):
- # look for literal prefix
- prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags)
- # if no prefix, look for charset prefix
- if not prefix:
- charset = _get_charset_prefix(pattern, flags)
-## if prefix:
-## print("*** PREFIX", prefix, prefix_skip)
-## if charset:
-## print("*** CHARSET", charset)
- # add an info block
- emit = code.append
- emit(INFO)
- skip = len(code); emit(0)
- # literal flag
- mask = 0
- if prefix:
- mask = SRE_INFO_PREFIX
- if prefix_skip is None and got_all:
- mask = mask | SRE_INFO_LITERAL
- elif charset:
- mask = mask | SRE_INFO_CHARSET
- emit(mask)
- # pattern length
- if lo < MAXCODE:
- emit(lo)
- else:
- emit(MAXCODE)
- prefix = prefix[:MAXCODE]
- emit(min(hi, MAXCODE))
- # add literal prefix
- if prefix:
- emit(len(prefix)) # length
- if prefix_skip is None:
- prefix_skip = len(prefix)
- emit(prefix_skip) # skip
- code.extend(prefix)
- # generate overlap table
- code.extend(_generate_overlap_table(prefix))
- elif charset:
- charset, hascased = _optimize_charset(charset)
- assert not hascased
- _compile_charset(charset, flags, code)
- code[skip] = len(code) - skip
-
-def isstring(obj):
- return isinstance(obj, (str, bytes))
-
-def _code(p, flags):
-
- flags = p.state.flags | flags
- code = []
-
- # compile info block
- _compile_info(code, p, flags)
-
- # compile the pattern
- _compile(code, p.data, flags)
-
- code.append(SUCCESS)
-
- return code
-
-def _hex_code(code):
- return '[%s]' % ', '.join('%#0*x' % (_sre.CODESIZE*2+2, x) for x in code)
-
-def dis(code):
- import sys
-
- labels = set()
- level = 0
- offset_width = len(str(len(code) - 1))
-
- def dis_(start, end):
- def print_(*args, to=None):
- if to is not None:
- labels.add(to)
- args += ('(to %d)' % (to,),)
- print('%*d%s ' % (offset_width, start, ':' if start in labels else '.'),
- end=' '*(level-1))
- print(*args)
-
- def print_2(*args):
- print(end=' '*(offset_width + 2*level))
- print(*args)
-
- nonlocal level
- level += 1
- i = start
- while i < end:
- start = i
- op = code[i]
- i += 1
- op = OPCODES[op]
- if op in (SUCCESS, FAILURE, ANY, ANY_ALL,
- MAX_UNTIL, MIN_UNTIL, NEGATE):
- print_(op)
- elif op in (LITERAL, NOT_LITERAL,
- LITERAL_IGNORE, NOT_LITERAL_IGNORE,
- LITERAL_UNI_IGNORE, NOT_LITERAL_UNI_IGNORE,
- LITERAL_LOC_IGNORE, NOT_LITERAL_LOC_IGNORE):
- arg = code[i]
- i += 1
- print_(op, '%#02x (%r)' % (arg, chr(arg)))
- elif op is AT:
- arg = code[i]
- i += 1
- arg = str(ATCODES[arg])
- assert arg[:3] == 'AT_'
- print_(op, arg[3:])
- elif op is CATEGORY:
- arg = code[i]
- i += 1
- arg = str(CHCODES[arg])
- assert arg[:9] == 'CATEGORY_'
- print_(op, arg[9:])
- elif op in (IN, IN_IGNORE, IN_UNI_IGNORE, IN_LOC_IGNORE):
- skip = code[i]
- print_(op, skip, to=i+skip)
- dis_(i+1, i+skip)
- i += skip
- elif op in (RANGE, RANGE_UNI_IGNORE):
- lo, hi = code[i: i+2]
- i += 2
- print_(op, '%#02x %#02x (%r-%r)' % (lo, hi, chr(lo), chr(hi)))
- elif op is CHARSET:
- print_(op, _hex_code(code[i: i + 256//_CODEBITS]))
- i += 256//_CODEBITS
- elif op is BIGCHARSET:
- arg = code[i]
- i += 1
- mapping = list(b''.join(x.to_bytes(_sre.CODESIZE, sys.byteorder)
- for x in code[i: i + 256//_sre.CODESIZE]))
- print_(op, arg, mapping)
- i += 256//_sre.CODESIZE
- level += 1
- for j in range(arg):
- print_2(_hex_code(code[i: i + 256//_CODEBITS]))
- i += 256//_CODEBITS
- level -= 1
- elif op in (MARK, GROUPREF, GROUPREF_IGNORE, GROUPREF_UNI_IGNORE,
- GROUPREF_LOC_IGNORE):
- arg = code[i]
- i += 1
- print_(op, arg)
- elif op is JUMP:
- skip = code[i]
- print_(op, skip, to=i+skip)
- i += 1
- elif op is BRANCH:
- skip = code[i]
- print_(op, skip, to=i+skip)
- while skip:
- dis_(i+1, i+skip)
- i += skip
- start = i
- skip = code[i]
- if skip:
- print_('branch', skip, to=i+skip)
- else:
- print_(FAILURE)
- i += 1
- elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE):
- skip, min, max = code[i: i+3]
- if max == MAXREPEAT:
- max = 'MAXREPEAT'
- print_(op, skip, min, max, to=i+skip)
- dis_(i+3, i+skip)
- i += skip
- elif op is GROUPREF_EXISTS:
- arg, skip = code[i: i+2]
- print_(op, arg, skip, to=i+skip)
- i += 2
- elif op in (ASSERT, ASSERT_NOT):
- skip, arg = code[i: i+2]
- print_(op, skip, arg, to=i+skip)
- dis_(i+2, i+skip)
- i += skip
- elif op is INFO:
- skip, flags, min, max = code[i: i+4]
- if max == MAXREPEAT:
- max = 'MAXREPEAT'
- print_(op, skip, bin(flags), min, max, to=i+skip)
- start = i+4
- if flags & SRE_INFO_PREFIX:
- prefix_len, prefix_skip = code[i+4: i+6]
- print_2(' prefix_skip', prefix_skip)
- start = i + 6
- prefix = code[start: start+prefix_len]
- print_2(' prefix',
- '[%s]' % ', '.join('%#02x' % x for x in prefix),
- '(%r)' % ''.join(map(chr, prefix)))
- start += prefix_len
- print_2(' overlap', code[start: start+prefix_len])
- start += prefix_len
- if flags & SRE_INFO_CHARSET:
- level += 1
- print_2('in')
- dis_(start, i+skip)
- level -= 1
- i += skip
- else:
- raise ValueError(op)
-
- level -= 1
-
- dis_(0, len(code))
-
-
-def compile(p, flags=0):
- # internal: convert pattern list to internal format
-
- if isstring(p):
- pattern = p
- p = sre_parse.parse(p, flags)
- else:
- pattern = None
-
- code = _code(p, flags)
-
- if flags & SRE_FLAG_DEBUG:
- print()
- dis(code)
-
- # map in either direction
- groupindex = p.state.groupdict
- indexgroup = [None] * p.state.groups
- for k, i in groupindex.items():
- indexgroup[i] = k
-
- return _sre.compile(
- pattern, flags | p.state.flags, code,
- p.state.groups-1,
- groupindex, tuple(indexgroup)
- )
+from re import _compiler as _
+globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'})
diff --git a/common/py3-stdlib/sre_constants.py b/common/py3-stdlib/sre_constants.py
index db3ca51..fa09d04 100644
--- a/common/py3-stdlib/sre_constants.py
+++ b/common/py3-stdlib/sre_constants.py
@@ -1,261 +1,7 @@
-#
-# Secret Labs' Regular Expression Engine
-#
-# various symbols used by the regular expression engine.
-# run this script to update the _sre include files!
-#
-# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.
-#
-# See the sre.py file for information on usage and redistribution.
-#
+import warnings
+warnings.warn(f"module {__name__!r} is deprecated",
+ DeprecationWarning,
+ stacklevel=2)
-"""Internal support module for sre"""
-
-# update when constants are added or removed
-
-MAGIC = 20171005
-
-from _sre import MAXREPEAT, MAXGROUPS
-
-# SRE standard exception (access as sre.error)
-# should this really be here?
-
-class error(Exception):
- """Exception raised for invalid regular expressions.
-
- Attributes:
-
- msg: The unformatted error message
- pattern: The regular expression pattern
- pos: The index in the pattern where compilation failed (may be None)
- lineno: The line corresponding to pos (may be None)
- colno: The column corresponding to pos (may be None)
- """
-
- __module__ = 're'
-
- def __init__(self, msg, pattern=None, pos=None):
- self.msg = msg
- self.pattern = pattern
- self.pos = pos
- if pattern is not None and pos is not None:
- msg = '%s at position %d' % (msg, pos)
- if isinstance(pattern, str):
- newline = '\n'
- else:
- newline = b'\n'
- self.lineno = pattern.count(newline, 0, pos) + 1
- self.colno = pos - pattern.rfind(newline, 0, pos)
- if newline in pattern:
- msg = '%s (line %d, column %d)' % (msg, self.lineno, self.colno)
- else:
- self.lineno = self.colno = None
- super().__init__(msg)
-
-
-class _NamedIntConstant(int):
- def __new__(cls, value, name):
- self = super(_NamedIntConstant, cls).__new__(cls, value)
- self.name = name
- return self
-
- def __repr__(self):
- return self.name
-
- __reduce__ = None
-
-MAXREPEAT = _NamedIntConstant(MAXREPEAT, 'MAXREPEAT')
-
-def _makecodes(names):
- names = names.strip().split()
- items = [_NamedIntConstant(i, name) for i, name in enumerate(names)]
- globals().update({item.name: item for item in items})
- return items
-
-# operators
-# failure=0 success=1 (just because it looks better that way :-)
-OPCODES = _makecodes("""
- FAILURE SUCCESS
-
- ANY ANY_ALL
- ASSERT ASSERT_NOT
- AT
- BRANCH
- CALL
- CATEGORY
- CHARSET BIGCHARSET
- GROUPREF GROUPREF_EXISTS
- IN
- INFO
- JUMP
- LITERAL
- MARK
- MAX_UNTIL
- MIN_UNTIL
- NOT_LITERAL
- NEGATE
- RANGE
- REPEAT
- REPEAT_ONE
- SUBPATTERN
- MIN_REPEAT_ONE
-
- GROUPREF_IGNORE
- IN_IGNORE
- LITERAL_IGNORE
- NOT_LITERAL_IGNORE
-
- GROUPREF_LOC_IGNORE
- IN_LOC_IGNORE
- LITERAL_LOC_IGNORE
- NOT_LITERAL_LOC_IGNORE
-
- GROUPREF_UNI_IGNORE
- IN_UNI_IGNORE
- LITERAL_UNI_IGNORE
- NOT_LITERAL_UNI_IGNORE
- RANGE_UNI_IGNORE
-
- MIN_REPEAT MAX_REPEAT
-""")
-del OPCODES[-2:] # remove MIN_REPEAT and MAX_REPEAT
-
-# positions
-ATCODES = _makecodes("""
- AT_BEGINNING AT_BEGINNING_LINE AT_BEGINNING_STRING
- AT_BOUNDARY AT_NON_BOUNDARY
- AT_END AT_END_LINE AT_END_STRING
-
- AT_LOC_BOUNDARY AT_LOC_NON_BOUNDARY
-
- AT_UNI_BOUNDARY AT_UNI_NON_BOUNDARY
-""")
-
-# categories
-CHCODES = _makecodes("""
- CATEGORY_DIGIT CATEGORY_NOT_DIGIT
- CATEGORY_SPACE CATEGORY_NOT_SPACE
- CATEGORY_WORD CATEGORY_NOT_WORD
- CATEGORY_LINEBREAK CATEGORY_NOT_LINEBREAK
-
- CATEGORY_LOC_WORD CATEGORY_LOC_NOT_WORD
-
- CATEGORY_UNI_DIGIT CATEGORY_UNI_NOT_DIGIT
- CATEGORY_UNI_SPACE CATEGORY_UNI_NOT_SPACE
- CATEGORY_UNI_WORD CATEGORY_UNI_NOT_WORD
- CATEGORY_UNI_LINEBREAK CATEGORY_UNI_NOT_LINEBREAK
-""")
-
-
-# replacement operations for "ignore case" mode
-OP_IGNORE = {
- LITERAL: LITERAL_IGNORE,
- NOT_LITERAL: NOT_LITERAL_IGNORE,
-}
-
-OP_LOCALE_IGNORE = {
- LITERAL: LITERAL_LOC_IGNORE,
- NOT_LITERAL: NOT_LITERAL_LOC_IGNORE,
-}
-
-OP_UNICODE_IGNORE = {
- LITERAL: LITERAL_UNI_IGNORE,
- NOT_LITERAL: NOT_LITERAL_UNI_IGNORE,
-}
-
-AT_MULTILINE = {
- AT_BEGINNING: AT_BEGINNING_LINE,
- AT_END: AT_END_LINE
-}
-
-AT_LOCALE = {
- AT_BOUNDARY: AT_LOC_BOUNDARY,
- AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY
-}
-
-AT_UNICODE = {
- AT_BOUNDARY: AT_UNI_BOUNDARY,
- AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY
-}
-
-CH_LOCALE = {
- CATEGORY_DIGIT: CATEGORY_DIGIT,
- CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT,
- CATEGORY_SPACE: CATEGORY_SPACE,
- CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE,
- CATEGORY_WORD: CATEGORY_LOC_WORD,
- CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD,
- CATEGORY_LINEBREAK: CATEGORY_LINEBREAK,
- CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK
-}
-
-CH_UNICODE = {
- CATEGORY_DIGIT: CATEGORY_UNI_DIGIT,
- CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT,
- CATEGORY_SPACE: CATEGORY_UNI_SPACE,
- CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE,
- CATEGORY_WORD: CATEGORY_UNI_WORD,
- CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD,
- CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK,
- CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK
-}
-
-# flags
-SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking)
-SRE_FLAG_IGNORECASE = 2 # case insensitive
-SRE_FLAG_LOCALE = 4 # honour system locale
-SRE_FLAG_MULTILINE = 8 # treat target as multiline string
-SRE_FLAG_DOTALL = 16 # treat target as a single string
-SRE_FLAG_UNICODE = 32 # use unicode "locale"
-SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments
-SRE_FLAG_DEBUG = 128 # debugging
-SRE_FLAG_ASCII = 256 # use ascii "locale"
-
-# flags for INFO primitive
-SRE_INFO_PREFIX = 1 # has prefix
-SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix)
-SRE_INFO_CHARSET = 4 # pattern starts with character from given set
-
-if __name__ == "__main__":
- def dump(f, d, prefix):
- items = sorted(d)
- for item in items:
- f.write("#define %s_%s %d\n" % (prefix, item, item))
- with open("sre_constants.h", "w") as f:
- f.write("""\
-/*
- * Secret Labs' Regular Expression Engine
- *
- * regular expression matching engine
- *
- * NOTE: This file is generated by sre_constants.py. If you need
- * to change anything in here, edit sre_constants.py and run it.
- *
- * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved.
- *
- * See the _sre.c file for information on usage and redistribution.
- */
-
-""")
-
- f.write("#define SRE_MAGIC %d\n" % MAGIC)
-
- dump(f, OPCODES, "SRE_OP")
- dump(f, ATCODES, "SRE")
- dump(f, CHCODES, "SRE")
-
- f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE)
- f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE)
- f.write("#define SRE_FLAG_LOCALE %d\n" % SRE_FLAG_LOCALE)
- f.write("#define SRE_FLAG_MULTILINE %d\n" % SRE_FLAG_MULTILINE)
- f.write("#define SRE_FLAG_DOTALL %d\n" % SRE_FLAG_DOTALL)
- f.write("#define SRE_FLAG_UNICODE %d\n" % SRE_FLAG_UNICODE)
- f.write("#define SRE_FLAG_VERBOSE %d\n" % SRE_FLAG_VERBOSE)
- f.write("#define SRE_FLAG_DEBUG %d\n" % SRE_FLAG_DEBUG)
- f.write("#define SRE_FLAG_ASCII %d\n" % SRE_FLAG_ASCII)
-
- f.write("#define SRE_INFO_PREFIX %d\n" % SRE_INFO_PREFIX)
- f.write("#define SRE_INFO_LITERAL %d\n" % SRE_INFO_LITERAL)
- f.write("#define SRE_INFO_CHARSET %d\n" % SRE_INFO_CHARSET)
-
- print("done")
+from re import _constants as _
+globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'})
diff --git a/common/py3-stdlib/sre_parse.py b/common/py3-stdlib/sre_parse.py
index 20a6025..25a3f55 100644
--- a/common/py3-stdlib/sre_parse.py
+++ b/common/py3-stdlib/sre_parse.py
@@ -1,1076 +1,7 @@
-#
-# Secret Labs' Regular Expression Engine
-#
-# convert re-style regular expression to sre pattern
-#
-# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.
-#
-# See the sre.py file for information on usage and redistribution.
-#
+import warnings
+warnings.warn(f"module {__name__!r} is deprecated",
+ DeprecationWarning,
+ stacklevel=2)
-"""Internal support module for sre"""
-
-# XXX: show string offset and offending character for all errors
-
-from sre_constants import *
-
-SPECIAL_CHARS = ".\\[{()*+?^$|"
-REPEAT_CHARS = "*+?{"
-
-DIGITS = frozenset("0123456789")
-
-OCTDIGITS = frozenset("01234567")
-HEXDIGITS = frozenset("0123456789abcdefABCDEF")
-ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-
-WHITESPACE = frozenset(" \t\n\r\v\f")
-
-_REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT})
-_UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY})
-
-ESCAPES = {
- r"\a": (LITERAL, ord("\a")),
- r"\b": (LITERAL, ord("\b")),
- r"\f": (LITERAL, ord("\f")),
- r"\n": (LITERAL, ord("\n")),
- r"\r": (LITERAL, ord("\r")),
- r"\t": (LITERAL, ord("\t")),
- r"\v": (LITERAL, ord("\v")),
- r"\\": (LITERAL, ord("\\"))
-}
-
-CATEGORIES = {
- r"\A": (AT, AT_BEGINNING_STRING), # start of string
- r"\b": (AT, AT_BOUNDARY),
- r"\B": (AT, AT_NON_BOUNDARY),
- r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]),
- r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]),
- r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]),
- r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]),
- r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]),
- r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]),
- r"\Z": (AT, AT_END_STRING), # end of string
-}
-
-FLAGS = {
- # standard flags
- "i": SRE_FLAG_IGNORECASE,
- "L": SRE_FLAG_LOCALE,
- "m": SRE_FLAG_MULTILINE,
- "s": SRE_FLAG_DOTALL,
- "x": SRE_FLAG_VERBOSE,
- # extensions
- "a": SRE_FLAG_ASCII,
- "t": SRE_FLAG_TEMPLATE,
- "u": SRE_FLAG_UNICODE,
-}
-
-TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE
-GLOBAL_FLAGS = SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE
-
-class Verbose(Exception):
- pass
-
-class State:
- # keeps track of state for parsing
- def __init__(self):
- self.flags = 0
- self.groupdict = {}
- self.groupwidths = [None] # group 0
- self.lookbehindgroups = None
- self.grouprefpos = {}
- @property
- def groups(self):
- return len(self.groupwidths)
- def opengroup(self, name=None):
- gid = self.groups
- self.groupwidths.append(None)
- if self.groups > MAXGROUPS:
- raise error("too many groups")
- if name is not None:
- ogid = self.groupdict.get(name, None)
- if ogid is not None:
- raise error("redefinition of group name %r as group %d; "
- "was group %d" % (name, gid, ogid))
- self.groupdict[name] = gid
- return gid
- def closegroup(self, gid, p):
- self.groupwidths[gid] = p.getwidth()
- def checkgroup(self, gid):
- return gid < self.groups and self.groupwidths[gid] is not None
-
- def checklookbehindgroup(self, gid, source):
- if self.lookbehindgroups is not None:
- if not self.checkgroup(gid):
- raise source.error('cannot refer to an open group')
- if gid >= self.lookbehindgroups:
- raise source.error('cannot refer to group defined in the same '
- 'lookbehind subpattern')
-
-class SubPattern:
- # a subpattern, in intermediate form
- def __init__(self, state, data=None):
- self.state = state
- if data is None:
- data = []
- self.data = data
- self.width = None
-
- def dump(self, level=0):
- nl = True
- seqtypes = (tuple, list)
- for op, av in self.data:
- print(level*" " + str(op), end='')
- if op is IN:
- # member sublanguage
- print()
- for op, a in av:
- print((level+1)*" " + str(op), a)
- elif op is BRANCH:
- print()
- for i, a in enumerate(av[1]):
- if i:
- print(level*" " + "OR")
- a.dump(level+1)
- elif op is GROUPREF_EXISTS:
- condgroup, item_yes, item_no = av
- print('', condgroup)
- item_yes.dump(level+1)
- if item_no:
- print(level*" " + "ELSE")
- item_no.dump(level+1)
- elif isinstance(av, seqtypes):
- nl = False
- for a in av:
- if isinstance(a, SubPattern):
- if not nl:
- print()
- a.dump(level+1)
- nl = True
- else:
- if not nl:
- print(' ', end='')
- print(a, end='')
- nl = False
- if not nl:
- print()
- else:
- print('', av)
- def __repr__(self):
- return repr(self.data)
- def __len__(self):
- return len(self.data)
- def __delitem__(self, index):
- del self.data[index]
- def __getitem__(self, index):
- if isinstance(index, slice):
- return SubPattern(self.state, self.data[index])
- return self.data[index]
- def __setitem__(self, index, code):
- self.data[index] = code
- def insert(self, index, code):
- self.data.insert(index, code)
- def append(self, code):
- self.data.append(code)
- def getwidth(self):
- # determine the width (min, max) for this subpattern
- if self.width is not None:
- return self.width
- lo = hi = 0
- for op, av in self.data:
- if op is BRANCH:
- i = MAXREPEAT - 1
- j = 0
- for av in av[1]:
- l, h = av.getwidth()
- i = min(i, l)
- j = max(j, h)
- lo = lo + i
- hi = hi + j
- elif op is CALL:
- i, j = av.getwidth()
- lo = lo + i
- hi = hi + j
- elif op is SUBPATTERN:
- i, j = av[-1].getwidth()
- lo = lo + i
- hi = hi + j
- elif op in _REPEATCODES:
- i, j = av[2].getwidth()
- lo = lo + i * av[0]
- hi = hi + j * av[1]
- elif op in _UNITCODES:
- lo = lo + 1
- hi = hi + 1
- elif op is GROUPREF:
- i, j = self.state.groupwidths[av]
- lo = lo + i
- hi = hi + j
- elif op is GROUPREF_EXISTS:
- i, j = av[1].getwidth()
- if av[2] is not None:
- l, h = av[2].getwidth()
- i = min(i, l)
- j = max(j, h)
- else:
- i = 0
- lo = lo + i
- hi = hi + j
- elif op is SUCCESS:
- break
- self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT)
- return self.width
-
-class Tokenizer:
- def __init__(self, string):
- self.istext = isinstance(string, str)
- self.string = string
- if not self.istext:
- string = str(string, 'latin1')
- self.decoded_string = string
- self.index = 0
- self.next = None
- self.__next()
- def __next(self):
- index = self.index
- try:
- char = self.decoded_string[index]
- except IndexError:
- self.next = None
- return
- if char == "\\":
- index += 1
- try:
- char += self.decoded_string[index]
- except IndexError:
- raise error("bad escape (end of pattern)",
- self.string, len(self.string) - 1) from None
- self.index = index + 1
- self.next = char
- def match(self, char):
- if char == self.next:
- self.__next()
- return True
- return False
- def get(self):
- this = self.next
- self.__next()
- return this
- def getwhile(self, n, charset):
- result = ''
- for _ in range(n):
- c = self.next
- if c not in charset:
- break
- result += c
- self.__next()
- return result
- def getuntil(self, terminator, name):
- result = ''
- while True:
- c = self.next
- self.__next()
- if c is None:
- if not result:
- raise self.error("missing " + name)
- raise self.error("missing %s, unterminated name" % terminator,
- len(result))
- if c == terminator:
- if not result:
- raise self.error("missing " + name, 1)
- break
- result += c
- return result
- @property
- def pos(self):
- return self.index - len(self.next or '')
- def tell(self):
- return self.index - len(self.next or '')
- def seek(self, index):
- self.index = index
- self.__next()
-
- def error(self, msg, offset=0):
- return error(msg, self.string, self.tell() - offset)
-
-def _class_escape(source, escape):
- # handle escape code inside character class
- code = ESCAPES.get(escape)
- if code:
- return code
- code = CATEGORIES.get(escape)
- if code and code[0] is IN:
- return code
- try:
- c = escape[1:2]
- if c == "x":
- # hexadecimal escape (exactly two digits)
- escape += source.getwhile(2, HEXDIGITS)
- if len(escape) != 4:
- raise source.error("incomplete escape %s" % escape, len(escape))
- return LITERAL, int(escape[2:], 16)
- elif c == "u" and source.istext:
- # unicode escape (exactly four digits)
- escape += source.getwhile(4, HEXDIGITS)
- if len(escape) != 6:
- raise source.error("incomplete escape %s" % escape, len(escape))
- return LITERAL, int(escape[2:], 16)
- elif c == "U" and source.istext:
- # unicode escape (exactly eight digits)
- escape += source.getwhile(8, HEXDIGITS)
- if len(escape) != 10:
- raise source.error("incomplete escape %s" % escape, len(escape))
- c = int(escape[2:], 16)
- chr(c) # raise ValueError for invalid code
- return LITERAL, c
- elif c == "N" and source.istext:
- import unicodedata
- # named unicode escape e.g. \N{EM DASH}
- if not source.match('{'):
- raise source.error("missing {")
- charname = source.getuntil('}', 'character name')
- try:
- c = ord(unicodedata.lookup(charname))
- except (KeyError, TypeError):
- raise source.error("undefined character name %r" % charname,
- len(charname) + len(r'\N{}'))
- return LITERAL, c
- elif c in OCTDIGITS:
- # octal escape (up to three digits)
- escape += source.getwhile(2, OCTDIGITS)
- c = int(escape[1:], 8)
- if c > 0o377:
- raise source.error('octal escape value %s outside of '
- 'range 0-0o377' % escape, len(escape))
- return LITERAL, c
- elif c in DIGITS:
- raise ValueError
- if len(escape) == 2:
- if c in ASCIILETTERS:
- raise source.error('bad escape %s' % escape, len(escape))
- return LITERAL, ord(escape[1])
- except ValueError:
- pass
- raise source.error("bad escape %s" % escape, len(escape))
-
-def _escape(source, escape, state):
- # handle escape code in expression
- code = CATEGORIES.get(escape)
- if code:
- return code
- code = ESCAPES.get(escape)
- if code:
- return code
- try:
- c = escape[1:2]
- if c == "x":
- # hexadecimal escape
- escape += source.getwhile(2, HEXDIGITS)
- if len(escape) != 4:
- raise source.error("incomplete escape %s" % escape, len(escape))
- return LITERAL, int(escape[2:], 16)
- elif c == "u" and source.istext:
- # unicode escape (exactly four digits)
- escape += source.getwhile(4, HEXDIGITS)
- if len(escape) != 6:
- raise source.error("incomplete escape %s" % escape, len(escape))
- return LITERAL, int(escape[2:], 16)
- elif c == "U" and source.istext:
- # unicode escape (exactly eight digits)
- escape += source.getwhile(8, HEXDIGITS)
- if len(escape) != 10:
- raise source.error("incomplete escape %s" % escape, len(escape))
- c = int(escape[2:], 16)
- chr(c) # raise ValueError for invalid code
- return LITERAL, c
- elif c == "N" and source.istext:
- import unicodedata
- # named unicode escape e.g. \N{EM DASH}
- if not source.match('{'):
- raise source.error("missing {")
- charname = source.getuntil('}', 'character name')
- try:
- c = ord(unicodedata.lookup(charname))
- except (KeyError, TypeError):
- raise source.error("undefined character name %r" % charname,
- len(charname) + len(r'\N{}'))
- return LITERAL, c
- elif c == "0":
- # octal escape
- escape += source.getwhile(2, OCTDIGITS)
- return LITERAL, int(escape[1:], 8)
- elif c in DIGITS:
- # octal escape *or* decimal group reference (sigh)
- if source.next in DIGITS:
- escape += source.get()
- if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and
- source.next in OCTDIGITS):
- # got three octal digits; this is an octal escape
- escape += source.get()
- c = int(escape[1:], 8)
- if c > 0o377:
- raise source.error('octal escape value %s outside of '
- 'range 0-0o377' % escape,
- len(escape))
- return LITERAL, c
- # not an octal escape, so this is a group reference
- group = int(escape[1:])
- if group < state.groups:
- if not state.checkgroup(group):
- raise source.error("cannot refer to an open group",
- len(escape))
- state.checklookbehindgroup(group, source)
- return GROUPREF, group
- raise source.error("invalid group reference %d" % group, len(escape) - 1)
- if len(escape) == 2:
- if c in ASCIILETTERS:
- raise source.error("bad escape %s" % escape, len(escape))
- return LITERAL, ord(escape[1])
- except ValueError:
- pass
- raise source.error("bad escape %s" % escape, len(escape))
-
-def _uniq(items):
- return list(dict.fromkeys(items))
-
-def _parse_sub(source, state, verbose, nested):
- # parse an alternation: a|b|c
-
- items = []
- itemsappend = items.append
- sourcematch = source.match
- start = source.tell()
- while True:
- itemsappend(_parse(source, state, verbose, nested + 1,
- not nested and not items))
- if not sourcematch("|"):
- break
-
- if len(items) == 1:
- return items[0]
-
- subpattern = SubPattern(state)
-
- # check if all items share a common prefix
- while True:
- prefix = None
- for item in items:
- if not item:
- break
- if prefix is None:
- prefix = item[0]
- elif item[0] != prefix:
- break
- else:
- # all subitems start with a common "prefix".
- # move it out of the branch
- for item in items:
- del item[0]
- subpattern.append(prefix)
- continue # check next one
- break
-
- # check if the branch can be replaced by a character set
- set = []
- for item in items:
- if len(item) != 1:
- break
- op, av = item[0]
- if op is LITERAL:
- set.append((op, av))
- elif op is IN and av[0][0] is not NEGATE:
- set.extend(av)
- else:
- break
- else:
- # we can store this as a character set instead of a
- # branch (the compiler may optimize this even more)
- subpattern.append((IN, _uniq(set)))
- return subpattern
-
- subpattern.append((BRANCH, (None, items)))
- return subpattern
-
-def _parse(source, state, verbose, nested, first=False):
- # parse a simple pattern
- subpattern = SubPattern(state)
-
- # precompute constants into local variables
- subpatternappend = subpattern.append
- sourceget = source.get
- sourcematch = source.match
- _len = len
- _ord = ord
-
- while True:
-
- this = source.next
- if this is None:
- break # end of pattern
- if this in "|)":
- break # end of subpattern
- sourceget()
-
- if verbose:
- # skip whitespace and comments
- if this in WHITESPACE:
- continue
- if this == "#":
- while True:
- this = sourceget()
- if this is None or this == "\n":
- break
- continue
-
- if this[0] == "\\":
- code = _escape(source, this, state)
- subpatternappend(code)
-
- elif this not in SPECIAL_CHARS:
- subpatternappend((LITERAL, _ord(this)))
-
- elif this == "[":
- here = source.tell() - 1
- # character set
- set = []
- setappend = set.append
-## if sourcematch(":"):
-## pass # handle character classes
- if source.next == '[':
- import warnings
- warnings.warn(
- 'Possible nested set at position %d' % source.tell(),
- FutureWarning, stacklevel=nested + 6
- )
- negate = sourcematch("^")
- # check remaining characters
- while True:
- this = sourceget()
- if this is None:
- raise source.error("unterminated character set",
- source.tell() - here)
- if this == "]" and set:
- break
- elif this[0] == "\\":
- code1 = _class_escape(source, this)
- else:
- if set and this in '-&~|' and source.next == this:
- import warnings
- warnings.warn(
- 'Possible set %s at position %d' % (
- 'difference' if this == '-' else
- 'intersection' if this == '&' else
- 'symmetric difference' if this == '~' else
- 'union',
- source.tell() - 1),
- FutureWarning, stacklevel=nested + 6
- )
- code1 = LITERAL, _ord(this)
- if sourcematch("-"):
- # potential range
- that = sourceget()
- if that is None:
- raise source.error("unterminated character set",
- source.tell() - here)
- if that == "]":
- if code1[0] is IN:
- code1 = code1[1][0]
- setappend(code1)
- setappend((LITERAL, _ord("-")))
- break
- if that[0] == "\\":
- code2 = _class_escape(source, that)
- else:
- if that == '-':
- import warnings
- warnings.warn(
- 'Possible set difference at position %d' % (
- source.tell() - 2),
- FutureWarning, stacklevel=nested + 6
- )
- code2 = LITERAL, _ord(that)
- if code1[0] != LITERAL or code2[0] != LITERAL:
- msg = "bad character range %s-%s" % (this, that)
- raise source.error(msg, len(this) + 1 + len(that))
- lo = code1[1]
- hi = code2[1]
- if hi < lo:
- msg = "bad character range %s-%s" % (this, that)
- raise source.error(msg, len(this) + 1 + len(that))
- setappend((RANGE, (lo, hi)))
- else:
- if code1[0] is IN:
- code1 = code1[1][0]
- setappend(code1)
-
- set = _uniq(set)
- # XXX: <fl> should move set optimization to compiler!
- if _len(set) == 1 and set[0][0] is LITERAL:
- # optimization
- if negate:
- subpatternappend((NOT_LITERAL, set[0][1]))
- else:
- subpatternappend(set[0])
- else:
- if negate:
- set.insert(0, (NEGATE, None))
- # charmap optimization can't be added here because
- # global flags still are not known
- subpatternappend((IN, set))
-
- elif this in REPEAT_CHARS:
- # repeat previous item
- here = source.tell()
- if this == "?":
- min, max = 0, 1
- elif this == "*":
- min, max = 0, MAXREPEAT
-
- elif this == "+":
- min, max = 1, MAXREPEAT
- elif this == "{":
- if source.next == "}":
- subpatternappend((LITERAL, _ord(this)))
- continue
-
- min, max = 0, MAXREPEAT
- lo = hi = ""
- while source.next in DIGITS:
- lo += sourceget()
- if sourcematch(","):
- while source.next in DIGITS:
- hi += sourceget()
- else:
- hi = lo
- if not sourcematch("}"):
- subpatternappend((LITERAL, _ord(this)))
- source.seek(here)
- continue
-
- if lo:
- min = int(lo)
- if min >= MAXREPEAT:
- raise OverflowError("the repetition number is too large")
- if hi:
- max = int(hi)
- if max >= MAXREPEAT:
- raise OverflowError("the repetition number is too large")
- if max < min:
- raise source.error("min repeat greater than max repeat",
- source.tell() - here)
- else:
- raise AssertionError("unsupported quantifier %r" % (char,))
- # figure out which item to repeat
- if subpattern:
- item = subpattern[-1:]
- else:
- item = None
- if not item or item[0][0] is AT:
- raise source.error("nothing to repeat",
- source.tell() - here + len(this))
- if item[0][0] in _REPEATCODES:
- raise source.error("multiple repeat",
- source.tell() - here + len(this))
- if item[0][0] is SUBPATTERN:
- group, add_flags, del_flags, p = item[0][1]
- if group is None and not add_flags and not del_flags:
- item = p
- if sourcematch("?"):
- subpattern[-1] = (MIN_REPEAT, (min, max, item))
- else:
- subpattern[-1] = (MAX_REPEAT, (min, max, item))
-
- elif this == ".":
- subpatternappend((ANY, None))
-
- elif this == "(":
- start = source.tell() - 1
- group = True
- name = None
- add_flags = 0
- del_flags = 0
- if sourcematch("?"):
- # options
- char = sourceget()
- if char is None:
- raise source.error("unexpected end of pattern")
- if char == "P":
- # python extensions
- if sourcematch("<"):
- # named group: skip forward to end of name
- name = source.getuntil(">", "group name")
- if not name.isidentifier():
- msg = "bad character in group name %r" % name
- raise source.error(msg, len(name) + 1)
- elif sourcematch("="):
- # named backreference
- name = source.getuntil(")", "group name")
- if not name.isidentifier():
- msg = "bad character in group name %r" % name
- raise source.error(msg, len(name) + 1)
- gid = state.groupdict.get(name)
- if gid is None:
- msg = "unknown group name %r" % name
- raise source.error(msg, len(name) + 1)
- if not state.checkgroup(gid):
- raise source.error("cannot refer to an open group",
- len(name) + 1)
- state.checklookbehindgroup(gid, source)
- subpatternappend((GROUPREF, gid))
- continue
-
- else:
- char = sourceget()
- if char is None:
- raise source.error("unexpected end of pattern")
- raise source.error("unknown extension ?P" + char,
- len(char) + 2)
- elif char == ":":
- # non-capturing group
- group = None
- elif char == "#":
- # comment
- while True:
- if source.next is None:
- raise source.error("missing ), unterminated comment",
- source.tell() - start)
- if sourceget() == ")":
- break
- continue
-
- elif char in "=!<":
- # lookahead assertions
- dir = 1
- if char == "<":
- char = sourceget()
- if char is None:
- raise source.error("unexpected end of pattern")
- if char not in "=!":
- raise source.error("unknown extension ?<" + char,
- len(char) + 2)
- dir = -1 # lookbehind
- lookbehindgroups = state.lookbehindgroups
- if lookbehindgroups is None:
- state.lookbehindgroups = state.groups
- p = _parse_sub(source, state, verbose, nested + 1)
- if dir < 0:
- if lookbehindgroups is None:
- state.lookbehindgroups = None
- if not sourcematch(")"):
- raise source.error("missing ), unterminated subpattern",
- source.tell() - start)
- if char == "=":
- subpatternappend((ASSERT, (dir, p)))
- else:
- subpatternappend((ASSERT_NOT, (dir, p)))
- continue
-
- elif char == "(":
- # conditional backreference group
- condname = source.getuntil(")", "group name")
- if condname.isidentifier():
- condgroup = state.groupdict.get(condname)
- if condgroup is None:
- msg = "unknown group name %r" % condname
- raise source.error(msg, len(condname) + 1)
- else:
- try:
- condgroup = int(condname)
- if condgroup < 0:
- raise ValueError
- except ValueError:
- msg = "bad character in group name %r" % condname
- raise source.error(msg, len(condname) + 1) from None
- if not condgroup:
- raise source.error("bad group number",
- len(condname) + 1)
- if condgroup >= MAXGROUPS:
- msg = "invalid group reference %d" % condgroup
- raise source.error(msg, len(condname) + 1)
- if condgroup not in state.grouprefpos:
- state.grouprefpos[condgroup] = (
- source.tell() - len(condname) - 1
- )
- state.checklookbehindgroup(condgroup, source)
- item_yes = _parse(source, state, verbose, nested + 1)
- if source.match("|"):
- item_no = _parse(source, state, verbose, nested + 1)
- if source.next == "|":
- raise source.error("conditional backref with more than two branches")
- else:
- item_no = None
- if not source.match(")"):
- raise source.error("missing ), unterminated subpattern",
- source.tell() - start)
- subpatternappend((GROUPREF_EXISTS, (condgroup, item_yes, item_no)))
- continue
-
- elif char in FLAGS or char == "-":
- # flags
- flags = _parse_flags(source, state, char)
- if flags is None: # global flags
- if not first or subpattern:
- import warnings
- warnings.warn(
- 'Flags not at the start of the expression %r%s'
- ' but at position %d' % (
- source.string[:20], # truncate long regexes
- ' (truncated)' if len(source.string) > 20 else '',
- start,
- ),
- DeprecationWarning, stacklevel=nested + 6
- )
- if (state.flags & SRE_FLAG_VERBOSE) and not verbose:
- raise Verbose
- continue
-
- add_flags, del_flags = flags
- group = None
- else:
- raise source.error("unknown extension ?" + char,
- len(char) + 1)
-
- # parse group contents
- if group is not None:
- try:
- group = state.opengroup(name)
- except error as err:
- raise source.error(err.msg, len(name) + 1) from None
- sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and
- not (del_flags & SRE_FLAG_VERBOSE))
- p = _parse_sub(source, state, sub_verbose, nested + 1)
- if not source.match(")"):
- raise source.error("missing ), unterminated subpattern",
- source.tell() - start)
- if group is not None:
- state.closegroup(group, p)
- subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p)))
-
- elif this == "^":
- subpatternappend((AT, AT_BEGINNING))
-
- elif this == "$":
- subpatternappend((AT, AT_END))
-
- else:
- raise AssertionError("unsupported special character %r" % (char,))
-
- # unpack non-capturing groups
- for i in range(len(subpattern))[::-1]:
- op, av = subpattern[i]
- if op is SUBPATTERN:
- group, add_flags, del_flags, p = av
- if group is None and not add_flags and not del_flags:
- subpattern[i: i+1] = p
-
- return subpattern
-
-def _parse_flags(source, state, char):
- sourceget = source.get
- add_flags = 0
- del_flags = 0
- if char != "-":
- while True:
- flag = FLAGS[char]
- if source.istext:
- if char == 'L':
- msg = "bad inline flags: cannot use 'L' flag with a str pattern"
- raise source.error(msg)
- else:
- if char == 'u':
- msg = "bad inline flags: cannot use 'u' flag with a bytes pattern"
- raise source.error(msg)
- add_flags |= flag
- if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag:
- msg = "bad inline flags: flags 'a', 'u' and 'L' are incompatible"
- raise source.error(msg)
- char = sourceget()
- if char is None:
- raise source.error("missing -, : or )")
- if char in ")-:":
- break
- if char not in FLAGS:
- msg = "unknown flag" if char.isalpha() else "missing -, : or )"
- raise source.error(msg, len(char))
- if char == ")":
- state.flags |= add_flags
- return None
- if add_flags & GLOBAL_FLAGS:
- raise source.error("bad inline flags: cannot turn on global flag", 1)
- if char == "-":
- char = sourceget()
- if char is None:
- raise source.error("missing flag")
- if char not in FLAGS:
- msg = "unknown flag" if char.isalpha() else "missing flag"
- raise source.error(msg, len(char))
- while True:
- flag = FLAGS[char]
- if flag & TYPE_FLAGS:
- msg = "bad inline flags: cannot turn off flags 'a', 'u' and 'L'"
- raise source.error(msg)
- del_flags |= flag
- char = sourceget()
- if char is None:
- raise source.error("missing :")
- if char == ":":
- break
- if char not in FLAGS:
- msg = "unknown flag" if char.isalpha() else "missing :"
- raise source.error(msg, len(char))
- assert char == ":"
- if del_flags & GLOBAL_FLAGS:
- raise source.error("bad inline flags: cannot turn off global flag", 1)
- if add_flags & del_flags:
- raise source.error("bad inline flags: flag turned on and off", 1)
- return add_flags, del_flags
-
-def fix_flags(src, flags):
- # Check and fix flags according to the type of pattern (str or bytes)
- if isinstance(src, str):
- if flags & SRE_FLAG_LOCALE:
- raise ValueError("cannot use LOCALE flag with a str pattern")
- if not flags & SRE_FLAG_ASCII:
- flags |= SRE_FLAG_UNICODE
- elif flags & SRE_FLAG_UNICODE:
- raise ValueError("ASCII and UNICODE flags are incompatible")
- else:
- if flags & SRE_FLAG_UNICODE:
- raise ValueError("cannot use UNICODE flag with a bytes pattern")
- if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII:
- raise ValueError("ASCII and LOCALE flags are incompatible")
- return flags
-
-def parse(str, flags=0, state=None):
- # parse 're' pattern into list of (opcode, argument) tuples
-
- source = Tokenizer(str)
-
- if state is None:
- state = State()
- state.flags = flags
- state.str = str
-
- try:
- p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
- except Verbose:
- # the VERBOSE flag was switched on inside the pattern. to be
- # on the safe side, we'll parse the whole thing again...
- state = State()
- state.flags = flags | SRE_FLAG_VERBOSE
- state.str = str
- source.seek(0)
- p = _parse_sub(source, state, True, 0)
-
- p.state.flags = fix_flags(str, p.state.flags)
-
- if source.next is not None:
- assert source.next == ")"
- raise source.error("unbalanced parenthesis")
-
- for g in p.state.grouprefpos:
- if g >= p.state.groups:
- msg = "invalid group reference %d" % g
- raise error(msg, str, p.state.grouprefpos[g])
-
- if flags & SRE_FLAG_DEBUG:
- p.dump()
-
- return p
-
-def parse_template(source, state):
- # parse 're' replacement string into list of literals and
- # group references
- s = Tokenizer(source)
- sget = s.get
- groups = []
- literals = []
- literal = []
- lappend = literal.append
- def addgroup(index, pos):
- if index > state.groups:
- raise s.error("invalid group reference %d" % index, pos)
- if literal:
- literals.append(''.join(literal))
- del literal[:]
- groups.append((len(literals), index))
- literals.append(None)
- groupindex = state.groupindex
- while True:
- this = sget()
- if this is None:
- break # end of replacement string
- if this[0] == "\\":
- # group
- c = this[1]
- if c == "g":
- name = ""
- if not s.match("<"):
- raise s.error("missing <")
- name = s.getuntil(">", "group name")
- if name.isidentifier():
- try:
- index = groupindex[name]
- except KeyError:
- raise IndexError("unknown group name %r" % name)
- else:
- try:
- index = int(name)
- if index < 0:
- raise ValueError
- except ValueError:
- raise s.error("bad character in group name %r" % name,
- len(name) + 1) from None
- if index >= MAXGROUPS:
- raise s.error("invalid group reference %d" % index,
- len(name) + 1)
- addgroup(index, len(name) + 1)
- elif c == "0":
- if s.next in OCTDIGITS:
- this += sget()
- if s.next in OCTDIGITS:
- this += sget()
- lappend(chr(int(this[1:], 8) & 0xff))
- elif c in DIGITS:
- isoctal = False
- if s.next in DIGITS:
- this += sget()
- if (c in OCTDIGITS and this[2] in OCTDIGITS and
- s.next in OCTDIGITS):
- this += sget()
- isoctal = True
- c = int(this[1:], 8)
- if c > 0o377:
- raise s.error('octal escape value %s outside of '
- 'range 0-0o377' % this, len(this))
- lappend(chr(c))
- if not isoctal:
- addgroup(int(this[1:]), len(this) - 1)
- else:
- try:
- this = chr(ESCAPES[this][1])
- except KeyError:
- if c in ASCIILETTERS:
- raise s.error('bad escape %s' % this, len(this))
- lappend(this)
- else:
- lappend(this)
- if literal:
- literals.append(''.join(literal))
- if not isinstance(source, str):
- # The tokenizer implicitly decodes bytes objects as latin-1, we must
- # therefore re-encode the final representation.
- literals = [None if s is None else s.encode('latin-1') for s in literals]
- return groups, literals
-
-def expand_template(template, match):
- g = match.group
- empty = match.string[:0]
- groups, literals = template
- literals = literals[:]
- try:
- for index, group in groups:
- literals[index] = g(group) or empty
- except IndexError:
- raise error("invalid group reference %d" % index)
- return empty.join(literals)
+from re import _parser as _
+globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'})
diff --git a/common/py3-stdlib/ssl.py b/common/py3-stdlib/ssl.py
index b09d684..ebac1d6 100644
--- a/common/py3-stdlib/ssl.py
+++ b/common/py3-stdlib/ssl.py
@@ -95,6 +95,7 @@
import os
from collections import namedtuple
from enum import Enum as _Enum, IntEnum as _IntEnum, IntFlag as _IntFlag
+from enum import _simple_enum
import _ssl # if we can't import it, let the error propagate
@@ -119,7 +120,6 @@
)
from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION
-
_IntEnum._convert_(
'_SSLMethod', __name__,
lambda name: name.startswith('PROTOCOL_') and name != 'PROTOCOL_SSLv23',
@@ -156,7 +156,8 @@
_SSLv2_IF_EXISTS = getattr(_SSLMethod, 'PROTOCOL_SSLv2', None)
-class TLSVersion(_IntEnum):
+@_simple_enum(_IntEnum)
+class TLSVersion:
MINIMUM_SUPPORTED = _ssl.PROTO_MINIMUM_SUPPORTED
SSLv3 = _ssl.PROTO_SSLv3
TLSv1 = _ssl.PROTO_TLSv1
@@ -166,7 +167,8 @@
MAXIMUM_SUPPORTED = _ssl.PROTO_MAXIMUM_SUPPORTED
-class _TLSContentType(_IntEnum):
+@_simple_enum(_IntEnum)
+class _TLSContentType:
"""Content types (record layer)
See RFC 8446, section B.1
@@ -180,7 +182,8 @@
INNER_CONTENT_TYPE = 0x101
-class _TLSAlertType(_IntEnum):
+@_simple_enum(_IntEnum)
+class _TLSAlertType:
"""Alert types for TLSContentType.ALERT messages
See RFC 8466, section B.2
@@ -221,7 +224,8 @@
NO_APPLICATION_PROTOCOL = 120
-class _TLSMessageType(_IntEnum):
+@_simple_enum(_IntEnum)
+class _TLSMessageType:
"""Message types (handshake protocol)
See RFC 8446, section B.3
@@ -276,7 +280,7 @@
def _dnsname_match(dn, hostname):
"""Matching according to RFC 6125, section 6.4.3
- - Hostnames are compared lower case.
+ - Hostnames are compared lower-case.
- For IDNA, both dn and hostname must be encoded as IDN A-label (ACE).
- Partial wildcards like 'www*.example.org', multiple wildcards, sole
wildcard or wildcards in labels other then the left-most label are not
@@ -364,7 +368,7 @@
(section 1.7.2 - "Out of Scope").
"""
# OpenSSL may add a trailing newline to a subjectAltName's IP address,
- # commonly woth IPv6 addresses. Strip off trailing \n.
+ # commonly with IPv6 addresses. Strip off trailing \n.
ip = _inet_paton(cert_ipaddress.rstrip())
return ip == host_ip
diff --git a/common/py3-stdlib/statistics.py b/common/py3-stdlib/statistics.py
index 52f1785..3b3b43b 100644
--- a/common/py3-stdlib/statistics.py
+++ b/common/py3-stdlib/statistics.py
@@ -130,14 +130,18 @@
import math
import numbers
import random
+import sys
from fractions import Fraction
from decimal import Decimal
from itertools import groupby, repeat
from bisect import bisect_left, bisect_right
from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum
-from operator import itemgetter
-from collections import Counter, namedtuple
+from functools import reduce
+from operator import mul
+from collections import Counter, namedtuple, defaultdict
+
+_SQRT2 = sqrt(2.0)
# === Exceptions ===
@@ -180,11 +184,12 @@
allowed.
"""
count = 0
+ types = set()
+ types_add = types.add
partials = {}
partials_get = partials.get
- T = int
for typ, values in groupby(data, type):
- T = _coerce(T, typ) # or raise TypeError
+ types_add(typ)
for n, d in map(_exact_ratio, values):
count += 1
partials[d] = partials_get(d, 0) + n
@@ -196,9 +201,51 @@
else:
# Sum all the partial sums using builtin sum.
total = sum(Fraction(n, d) for d, n in partials.items())
+ T = reduce(_coerce, types, int) # or raise TypeError
return (T, total, count)
+def _ss(data, c=None):
+ """Return the exact mean and sum of square deviations of sequence data.
+
+ Calculations are done in a single pass, allowing the input to be an iterator.
+
+ If given *c* is used the mean; otherwise, it is calculated from the data.
+ Use the *c* argument with care, as it can lead to garbage results.
+
+ """
+ if c is not None:
+ T, ssd, count = _sum((d := x - c) * d for x in data)
+ return (T, ssd, c, count)
+ count = 0
+ types = set()
+ types_add = types.add
+ sx_partials = defaultdict(int)
+ sxx_partials = defaultdict(int)
+ for typ, values in groupby(data, type):
+ types_add(typ)
+ for n, d in map(_exact_ratio, values):
+ count += 1
+ sx_partials[d] += n
+ sxx_partials[d] += n * n
+ if not count:
+ ssd = c = Fraction(0)
+ elif None in sx_partials:
+ # The sum will be a NAN or INF. We can ignore all the finite
+ # partials, and just look at this special one.
+ ssd = c = sx_partials[None]
+ assert not _isfinite(ssd)
+ else:
+ sx = sum(Fraction(n, d) for d, n in sx_partials.items())
+ sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items())
+ # This formula has poor numeric properties for floats,
+ # but with fractions it is exact.
+ ssd = (count * sxx - sx * sx) / count
+ c = sx / count
+ T = reduce(_coerce, types, int) # or raise TypeError
+ return (T, ssd, c, count)
+
+
def _isfinite(x):
try:
return x.is_finite() # Likely a Decimal.
@@ -245,6 +292,28 @@
x is expected to be an int, Fraction, Decimal or float.
"""
+
+ # XXX We should revisit whether using fractions to accumulate exact
+ # ratios is the right way to go.
+
+ # The integer ratios for binary floats can have numerators or
+ # denominators with over 300 decimal digits. The problem is more
+ # acute with decimal floats where the default decimal context
+ # supports a huge range of exponents from Emin=-999999 to
+ # Emax=999999. When expanded with as_integer_ratio(), numbers like
+ # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large
+ # numerators or denominators that will slow computation.
+
+ # When the integer ratios are accumulated as fractions, the size
+ # grows to cover the full range from the smallest magnitude to the
+ # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300),
+ # has a 616 digit numerator. Likewise,
+ # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000'))
+ # has 10,003 digit numerator.
+
+ # This doesn't seem to have been problem in practice, but it is a
+ # potential pitfall.
+
try:
return x.as_integer_ratio()
except AttributeError:
@@ -279,22 +348,6 @@
raise
-def _find_lteq(a, x):
- 'Locate the leftmost value exactly equal to x'
- i = bisect_left(a, x)
- if i != len(a) and a[i] == x:
- return i
- raise ValueError
-
-
-def _find_rteq(a, l, x):
- 'Locate the rightmost value exactly equal to x'
- i = bisect_right(a, x, lo=l)
- if i != (len(a) + 1) and a[i - 1] == x:
- return i - 1
- raise ValueError
-
-
def _fail_neg(values, errmsg='negative value'):
"""Iterate over values, failing if any are less than zero."""
for x in values:
@@ -303,6 +356,59 @@
yield x
+def _integer_sqrt_of_frac_rto(n: int, m: int) -> int:
+ """Square root of n/m, rounded to the nearest integer using round-to-odd."""
+ # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf
+ a = math.isqrt(n // m)
+ return a | (a*a*m != n)
+
+
+# For 53 bit precision floats, the bit width used in
+# _float_sqrt_of_frac() is 109.
+_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3
+
+
+def _float_sqrt_of_frac(n: int, m: int) -> float:
+ """Square root of n/m as a float, correctly rounded."""
+ # See principle and proof sketch at: https://bugs.python.org/msg407078
+ q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2
+ if q >= 0:
+ numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q
+ denominator = 1
+ else:
+ numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m)
+ denominator = 1 << -q
+ return numerator / denominator # Convert to float
+
+
+def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal:
+ """Square root of n/m as a Decimal, correctly rounded."""
+ # Premise: For decimal, computing (n/m).sqrt() can be off
+ # by 1 ulp from the correctly rounded result.
+ # Method: Check the result, moving up or down a step if needed.
+ if n <= 0:
+ if not n:
+ return Decimal('0.0')
+ n, m = -n, -m
+
+ root = (Decimal(n) / Decimal(m)).sqrt()
+ nr, dr = root.as_integer_ratio()
+
+ plus = root.next_plus()
+ np, dp = plus.as_integer_ratio()
+ # test: n / m > ((root + plus) / 2) ** 2
+ if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2:
+ return plus
+
+ minus = root.next_minus()
+ nm, dm = minus.as_integer_ratio()
+ # test: n / m < ((root + minus) / 2) ** 2
+ if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2:
+ return minus
+
+ return root
+
+
# === Measures of central tendency (averages) ===
def mean(data):
@@ -321,17 +427,13 @@
If ``data`` is empty, StatisticsError will be raised.
"""
- if iter(data) is data:
- data = list(data)
- n = len(data)
+ T, total, n = _sum(data)
if n < 1:
raise StatisticsError('mean requires at least one data point')
- T, total, count = _sum(data)
- assert count == n
return _convert(total / n, T)
-def fmean(data):
+def fmean(data, weights=None):
"""Convert data to floats and compute the arithmetic mean.
This runs faster than the mean() function and it always returns a float.
@@ -349,13 +451,24 @@
nonlocal n
for n, x in enumerate(iterable, start=1):
yield x
- total = fsum(count(data))
- else:
+ data = count(data)
+ if weights is None:
total = fsum(data)
- try:
+ if not n:
+ raise StatisticsError('fmean requires at least one data point')
return total / n
- except ZeroDivisionError:
- raise StatisticsError('fmean requires at least one data point') from None
+ try:
+ num_weights = len(weights)
+ except TypeError:
+ weights = list(weights)
+ num_weights = len(weights)
+ num = fsum(map(mul, data, weights))
+ if n != num_weights:
+ raise StatisticsError('data and weights must be the same length')
+ den = fsum(weights)
+ if not den:
+ raise StatisticsError('sum of weights must be non-zero')
+ return num / den
def geometric_mean(data):
@@ -498,58 +611,75 @@
return data[n // 2]
-def median_grouped(data, interval=1):
- """Return the 50th percentile (median) of grouped continuous data.
+def median_grouped(data, interval=1.0):
+ """Estimates the median for numeric data binned around the midpoints
+ of consecutive, fixed-width intervals.
- >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5])
- 3.7
- >>> median_grouped([52, 52, 53, 54])
- 52.5
+ The *data* can be any iterable of numeric data with each value being
+ exactly the midpoint of a bin. At least one value must be present.
- This calculates the median as the 50th percentile, and should be
- used when your data is continuous and grouped. In the above example,
- the values 1, 2, 3, etc. actually represent the midpoint of classes
- 0.5-1.5, 1.5-2.5, 2.5-3.5, etc. The middle value falls somewhere in
- class 3.5-4.5, and interpolation is used to estimate it.
+ The *interval* is width of each bin.
- Optional argument ``interval`` represents the class interval, and
- defaults to 1. Changing the class interval naturally will change the
- interpolated 50th percentile value:
+ For example, demographic information may have been summarized into
+ consecutive ten-year age groups with each group being represented
+ by the 5-year midpoints of the intervals:
- >>> median_grouped([1, 3, 3, 5, 7], interval=1)
- 3.25
- >>> median_grouped([1, 3, 3, 5, 7], interval=2)
- 3.5
+ >>> demographics = Counter({
+ ... 25: 172, # 20 to 30 years old
+ ... 35: 484, # 30 to 40 years old
+ ... 45: 387, # 40 to 50 years old
+ ... 55: 22, # 50 to 60 years old
+ ... 65: 6, # 60 to 70 years old
+ ... })
- This function does not check whether the data points are at least
- ``interval`` apart.
+ The 50th percentile (median) is the 536th person out of the 1071
+ member cohort. That person is in the 30 to 40 year old age group.
+
+ The regular median() function would assume that everyone in the
+ tricenarian age group was exactly 35 years old. A more tenable
+ assumption is that the 484 members of that age group are evenly
+ distributed between 30 and 40. For that, we use median_grouped().
+
+ >>> data = list(demographics.elements())
+ >>> median(data)
+ 35
+ >>> round(median_grouped(data, interval=10), 1)
+ 37.5
+
+ The caller is responsible for making sure the data points are separated
+ by exact multiples of *interval*. This is essential for getting a
+ correct result. The function does not check this precondition.
+
+ Inputs may be any numeric type that can be coerced to a float during
+ the interpolation step.
+
"""
data = sorted(data)
n = len(data)
- if n == 0:
+ if not n:
raise StatisticsError("no median for empty data")
- elif n == 1:
- return data[0]
- # Find the value at the midpoint. Remember this corresponds to the
- # centre of the class interval.
- x = data[n // 2]
- for obj in (x, interval):
- if isinstance(obj, (str, bytes)):
- raise TypeError('expected number but got %r' % obj)
- try:
- L = x - interval / 2 # The lower limit of the median interval.
- except TypeError:
- # Mixed type. For now we just coerce to float.
- L = float(x) - float(interval) / 2
- # Uses bisection search to search for x in data with log(n) time complexity
- # Find the position of leftmost occurrence of x in data
- l1 = _find_lteq(data, x)
- # Find the position of rightmost occurrence of x in data[l1...len(data)]
- # Assuming always l1 <= l2
- l2 = _find_rteq(data, l1, x)
- cf = l1
- f = l2 - l1 + 1
+ # Find the value at the midpoint. Remember this corresponds to the
+ # midpoint of the class interval.
+ x = data[n // 2]
+
+ # Using O(log n) bisection, find where all the x values occur in the data.
+ # All x will lie within data[i:j].
+ i = bisect_left(data, x)
+ j = bisect_right(data, x, lo=i)
+
+ # Coerce to floats, raising a TypeError if not possible
+ try:
+ interval = float(interval)
+ x = float(x)
+ except ValueError:
+ raise TypeError(f'Value cannot be converted to a float')
+
+ # Interpolate the median using the formula found at:
+ # https://www.cuemath.com/data/median-of-grouped-data/
+ L = x - interval / 2.0 # Lower limit of the median interval
+ cf = i # Cumulative frequency of the preceding interval
+ f = j - i # Number of elements in the median internal
return L + interval * (n / 2 - cf) / f
@@ -596,9 +726,11 @@
>>> multimode('')
[]
"""
- counts = Counter(iter(data)).most_common()
- maxcount, mode_items = next(groupby(counts, key=itemgetter(1)), (0, []))
- return list(map(itemgetter(0), mode_items))
+ counts = Counter(iter(data))
+ if not counts:
+ return []
+ maxcount = max(counts.values())
+ return [value for value, count in counts.items() if count == maxcount]
# Notes on methods for computing quantiles
@@ -685,41 +817,6 @@
# See http://mathworld.wolfram.com/Variance.html
# http://mathworld.wolfram.com/SampleVariance.html
-# http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
-#
-# Under no circumstances use the so-called "computational formula for
-# variance", as that is only suitable for hand calculations with a small
-# amount of low-precision data. It has terrible numeric properties.
-#
-# See a comparison of three computational methods here:
-# http://www.johndcook.com/blog/2008/09/26/comparing-three-methods-of-computing-standard-deviation/
-
-def _ss(data, c=None):
- """Return sum of square deviations of sequence data.
-
- If ``c`` is None, the mean is calculated in one pass, and the deviations
- from the mean are calculated in a second pass. Otherwise, deviations are
- calculated from ``c`` as given. Use the second case with care, as it can
- lead to garbage results.
- """
- if c is not None:
- T, total, count = _sum((x-c)**2 for x in data)
- return (T, total)
- T, total, count = _sum(data)
- mean_n, mean_d = (total / count).as_integer_ratio()
- partials = Counter()
- for n, d in map(_exact_ratio, data):
- diff_n = n * mean_d - d * mean_n
- diff_d = d * mean_d
- partials[diff_d * diff_d] += diff_n * diff_n
- if None in partials:
- # The sum will be a NAN or INF. We can ignore all the finite
- # partials, and just look at this special one.
- total = partials[None]
- assert not _isfinite(total)
- else:
- total = sum(Fraction(n, d) for d, n in partials.items())
- return (T, total)
def variance(data, xbar=None):
@@ -760,12 +857,9 @@
Fraction(67, 108)
"""
- if iter(data) is data:
- data = list(data)
- n = len(data)
+ T, ss, c, n = _ss(data, xbar)
if n < 2:
raise StatisticsError('variance requires at least two data points')
- T, ss = _ss(data, xbar)
return _convert(ss / (n - 1), T)
@@ -804,12 +898,9 @@
Fraction(13, 72)
"""
- if iter(data) is data:
- data = list(data)
- n = len(data)
+ T, ss, c, n = _ss(data, mu)
if n < 1:
raise StatisticsError('pvariance requires at least one data point')
- T, ss = _ss(data, mu)
return _convert(ss / n, T)
@@ -822,14 +913,13 @@
1.0810874155219827
"""
- # Fixme: Despite the exact sum of squared deviations, some inaccuracy
- # remain because there are two rounding steps. The first occurs in
- # the _convert() step for variance(), the second occurs in math.sqrt().
- var = variance(data, xbar)
- try:
- return var.sqrt()
- except AttributeError:
- return math.sqrt(var)
+ T, ss, c, n = _ss(data, xbar)
+ if n < 2:
+ raise StatisticsError('stdev requires at least two data points')
+ mss = ss / (n - 1)
+ if issubclass(T, Decimal):
+ return _decimal_sqrt_of_frac(mss.numerator, mss.denominator)
+ return _float_sqrt_of_frac(mss.numerator, mss.denominator)
def pstdev(data, mu=None):
@@ -841,14 +931,26 @@
0.986893273527251
"""
- # Fixme: Despite the exact sum of squared deviations, some inaccuracy
- # remain because there are two rounding steps. The first occurs in
- # the _convert() step for pvariance(), the second occurs in math.sqrt().
- var = pvariance(data, mu)
+ T, ss, c, n = _ss(data, mu)
+ if n < 1:
+ raise StatisticsError('pstdev requires at least one data point')
+ mss = ss / n
+ if issubclass(T, Decimal):
+ return _decimal_sqrt_of_frac(mss.numerator, mss.denominator)
+ return _float_sqrt_of_frac(mss.numerator, mss.denominator)
+
+
+def _mean_stdev(data):
+ """In one pass, compute the mean and sample standard deviation as floats."""
+ T, ss, xbar, n = _ss(data)
+ if n < 2:
+ raise StatisticsError('stdev requires at least two data points')
+ mss = ss / (n - 1)
try:
- return var.sqrt()
+ return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator)
except AttributeError:
- return math.sqrt(var)
+ # Handle Nans and Infs gracefully
+ return float(xbar), float(xbar) / float(ss)
# === Statistics for relations between two inputs ===
@@ -911,8 +1013,8 @@
xbar = fsum(x) / n
ybar = fsum(y) / n
sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))
- sxx = fsum((xi - xbar) ** 2.0 for xi in x)
- syy = fsum((yi - ybar) ** 2.0 for yi in y)
+ sxx = fsum((d := xi - xbar) * d for xi in x)
+ syy = fsum((d := yi - ybar) * d for yi in y)
try:
return sxy / sqrt(sxx * syy)
except ZeroDivisionError:
@@ -922,13 +1024,13 @@
LinearRegression = namedtuple('LinearRegression', ('slope', 'intercept'))
-def linear_regression(x, y, /):
+def linear_regression(x, y, /, *, proportional=False):
"""Slope and intercept for simple linear regression.
Return the slope and intercept of simple linear regression
parameters estimated using ordinary least squares. Simple linear
regression describes relationship between an independent variable
- *x* and a dependent variable *y* in terms of linear function:
+ *x* and a dependent variable *y* in terms of a linear function:
y = slope * x + intercept + noise
@@ -946,21 +1048,38 @@
>>> linear_regression(x, y) #doctest: +ELLIPSIS
LinearRegression(slope=3.09078914170..., intercept=1.75684970486...)
+ If *proportional* is true, the independent variable *x* and the
+ dependent variable *y* are assumed to be directly proportional.
+ The data is fit to a line passing through the origin.
+
+ Since the *intercept* will always be 0.0, the underlying linear
+ function simplifies to:
+
+ y = slope * x + noise
+
+ >>> y = [3 * x[i] + noise[i] for i in range(5)]
+ >>> linear_regression(x, y, proportional=True) #doctest: +ELLIPSIS
+ LinearRegression(slope=3.02447542484..., intercept=0.0)
+
"""
n = len(x)
if len(y) != n:
raise StatisticsError('linear regression requires that both inputs have same number of data points')
if n < 2:
raise StatisticsError('linear regression requires at least two data points')
- xbar = fsum(x) / n
- ybar = fsum(y) / n
- sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))
- sxx = fsum((xi - xbar) ** 2.0 for xi in x)
+ if proportional:
+ sxy = fsum(xi * yi for xi, yi in zip(x, y))
+ sxx = fsum(xi * xi for xi in x)
+ else:
+ xbar = fsum(x) / n
+ ybar = fsum(y) / n
+ sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))
+ sxx = fsum((d := xi - xbar) * d for xi in x)
try:
slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x)
except ZeroDivisionError:
raise StatisticsError('x is constant')
- intercept = ybar - slope * xbar
+ intercept = 0.0 if proportional else ybar - slope * xbar
return LinearRegression(slope=slope, intercept=intercept)
@@ -1068,10 +1187,7 @@
@classmethod
def from_samples(cls, data):
"Make a normal distribution instance from sample data."
- if not isinstance(data, (list, tuple)):
- data = list(data)
- xbar = fmean(data)
- return cls(xbar, stdev(data, xbar))
+ return cls(*_mean_stdev(data))
def samples(self, n, *, seed=None):
"Generate *n* samples for a given mean and standard deviation."
@@ -1081,16 +1197,17 @@
def pdf(self, x):
"Probability density function. P(x <= X < x+dx) / dx"
- variance = self._sigma ** 2.0
+ variance = self._sigma * self._sigma
if not variance:
raise StatisticsError('pdf() not defined when sigma is zero')
- return exp((x - self._mu)**2.0 / (-2.0*variance)) / sqrt(tau*variance)
+ diff = x - self._mu
+ return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance)
def cdf(self, x):
"Cumulative distribution function. P(X <= x)"
if not self._sigma:
raise StatisticsError('cdf() not defined when sigma is zero')
- return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * sqrt(2.0))))
+ return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2)))
def inv_cdf(self, p):
"""Inverse cumulative distribution function. x : P(X <= x) = p
@@ -1146,9 +1263,9 @@
dv = Y_var - X_var
dm = fabs(Y._mu - X._mu)
if not dv:
- return 1.0 - erf(dm / (2.0 * X._sigma * sqrt(2.0)))
+ return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2))
a = X._mu * Y_var - Y._mu * X_var
- b = X._sigma * Y._sigma * sqrt(dm**2.0 + dv * log(Y_var / X_var))
+ b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var))
x1 = (a + b) / dv
x2 = (a - b) / dv
return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2)))
@@ -1191,7 +1308,7 @@
@property
def variance(self):
"Square of the standard deviation."
- return self._sigma ** 2.0
+ return self._sigma * self._sigma
def __add__(x1, x2):
"""Add a constant or another NormalDist instance.
diff --git a/common/py3-stdlib/string.py b/common/py3-stdlib/string.py
index 489777b..2eab6d4 100644
--- a/common/py3-stdlib/string.py
+++ b/common/py3-stdlib/string.py
@@ -45,7 +45,7 @@
sep is used to split and join the words.
"""
- return (sep or ' ').join(x.capitalize() for x in s.split(sep))
+ return (sep or ' ').join(map(str.capitalize, s.split(sep)))
####################################################################
@@ -141,6 +141,35 @@
self.pattern)
return self.pattern.sub(convert, self.template)
+ def is_valid(self):
+ for mo in self.pattern.finditer(self.template):
+ if mo.group('invalid') is not None:
+ return False
+ if (mo.group('named') is None
+ and mo.group('braced') is None
+ and mo.group('escaped') is None):
+ # If all the groups are None, there must be
+ # another group we're not expecting
+ raise ValueError('Unrecognized named group in pattern',
+ self.pattern)
+ return True
+
+ def get_identifiers(self):
+ ids = []
+ for mo in self.pattern.finditer(self.template):
+ named = mo.group('named') or mo.group('braced')
+ if named is not None and named not in ids:
+ # add a named group only the first time it appears
+ ids.append(named)
+ elif (named is None
+ and mo.group('invalid') is None
+ and mo.group('escaped') is None):
+ # If all the groups are None, there must be
+ # another group we're not expecting
+ raise ValueError('Unrecognized named group in pattern',
+ self.pattern)
+ return ids
+
# Initialize Template.pattern. __init_subclass__() is automatically called
# only for subclasses, not for the Template class itself.
Template.__init_subclass__()
diff --git a/common/py3-stdlib/subprocess.py b/common/py3-stdlib/subprocess.py
index f1e3d64..fbc76b8 100644
--- a/common/py3-stdlib/subprocess.py
+++ b/common/py3-stdlib/subprocess.py
@@ -43,6 +43,7 @@
import builtins
import errno
import io
+import locale
import os
import time
import signal
@@ -65,16 +66,19 @@
# NOTE: We intentionally exclude list2cmdline as it is
# considered an internal implementation detail. issue10838.
+# use presence of msvcrt to detect Windows-like platforms (see bpo-8110)
try:
import msvcrt
- import _winapi
- _mswindows = True
except ModuleNotFoundError:
_mswindows = False
- import _posixsubprocess
- import select
- import selectors
else:
+ _mswindows = True
+
+# wasm32-emscripten and wasm32-wasi do not support processes
+_can_fork_exec = sys.platform not in {"emscripten", "wasi"}
+
+if _mswindows:
+ import _winapi
from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
STD_ERROR_HANDLE, SW_HIDE,
@@ -95,6 +99,24 @@
"NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS",
"CREATE_NO_WINDOW", "DETACHED_PROCESS",
"CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"])
+else:
+ if _can_fork_exec:
+ from _posixsubprocess import fork_exec as _fork_exec
+ # used in methods that are called by __del__
+ _waitpid = os.waitpid
+ _waitstatus_to_exitcode = os.waitstatus_to_exitcode
+ _WIFSTOPPED = os.WIFSTOPPED
+ _WSTOPSIG = os.WSTOPSIG
+ _WNOHANG = os.WNOHANG
+ else:
+ _fork_exec = None
+ _waitpid = None
+ _waitstatus_to_exitcode = None
+ _WIFSTOPPED = None
+ _WSTOPSIG = None
+ _WNOHANG = None
+ import select
+ import selectors
# Exception classes used by this module.
@@ -302,12 +324,14 @@
args.append('-E')
if sys.flags.no_user_site:
args.append('-s')
+ if sys.flags.safe_path:
+ args.append('-P')
# -W options
warnopts = sys.warnoptions[:]
- bytes_warning = sys.flags.bytes_warning
xoptions = getattr(sys, '_xoptions', {})
- dev_mode = ('dev' in xoptions)
+ bytes_warning = sys.flags.bytes_warning
+ dev_mode = sys.flags.dev_mode
if bytes_warning > 1:
warnopts.remove("error::BytesWarning")
@@ -334,6 +358,26 @@
return args
+def _text_encoding():
+ # Return default text encoding and emit EncodingWarning if
+ # sys.flags.warn_default_encoding is true.
+ if sys.flags.warn_default_encoding:
+ f = sys._getframe()
+ filename = f.f_code.co_filename
+ stacklevel = 2
+ while f := f.f_back:
+ if f.f_code.co_filename != filename:
+ break
+ stacklevel += 1
+ warnings.warn("'encoding' argument not specified.",
+ EncodingWarning, stacklevel)
+
+ if sys.flags.utf8_mode:
+ return "utf-8"
+ else:
+ return locale.getencoding()
+
+
def call(*popenargs, timeout=None, **kwargs):
"""Run command with arguments. Wait for command to complete or
timeout, then return the returncode attribute.
@@ -405,8 +449,9 @@
decoded according to locale encoding, or by "encoding" if set. Text mode
is triggered by setting any of text, encoding, errors or universal_newlines.
"""
- if 'stdout' in kwargs:
- raise ValueError('stdout argument not allowed, it will be overridden.')
+ for kw in ('stdout', 'check'):
+ if kw in kwargs:
+ raise ValueError(f'{kw} argument not allowed, it will be overridden.')
if 'input' in kwargs and kwargs['input'] is None:
# Explicitly passing input=None was previously equivalent to passing an
@@ -601,7 +646,7 @@
# Various tools for executing commands and looking at their output and status.
#
-def getstatusoutput(cmd):
+def getstatusoutput(cmd, *, encoding=None, errors=None):
"""Return (exitcode, output) of executing cmd in a shell.
Execute the string 'cmd' in a shell with 'check_output' and
@@ -623,7 +668,8 @@
(-15, '')
"""
try:
- data = check_output(cmd, shell=True, text=True, stderr=STDOUT)
+ data = check_output(cmd, shell=True, text=True, stderr=STDOUT,
+ encoding=encoding, errors=errors)
exitcode = 0
except CalledProcessError as ex:
data = ex.output
@@ -632,7 +678,7 @@
data = data[:-1]
return exitcode, data
-def getoutput(cmd):
+def getoutput(cmd, *, encoding=None, errors=None):
"""Return output (stdout or stderr) of executing cmd in a shell.
Like getstatusoutput(), except the exit status is ignored and the return
@@ -642,7 +688,8 @@
>>> subprocess.getoutput('ls /bin/ls')
'/bin/ls'
"""
- return getstatusoutput(cmd)[1]
+ return getstatusoutput(cmd, encoding=encoding, errors=errors)[1]
+
def _use_posix_spawn():
@@ -737,6 +784,8 @@
start_new_session (POSIX only)
+ process_group (POSIX only)
+
group (POSIX only)
extra_groups (POSIX only)
@@ -762,8 +811,14 @@
startupinfo=None, creationflags=0,
restore_signals=True, start_new_session=False,
pass_fds=(), *, user=None, group=None, extra_groups=None,
- encoding=None, errors=None, text=None, umask=-1, pipesize=-1):
+ encoding=None, errors=None, text=None, umask=-1, pipesize=-1,
+ process_group=None):
"""Create new Popen instance."""
+ if not _can_fork_exec:
+ raise OSError(
+ errno.ENOTSUP, f"{sys.platform} does not support processes."
+ )
+
_cleanup()
# Held while anything is calling waitpid before returncode has been
# updated to prevent clobbering returncode if wait() or poll() are
@@ -817,45 +872,9 @@
'and universal_newlines are supplied but '
'different. Pass one or the other.')
- # Input and output objects. The general principle is like
- # this:
- #
- # Parent Child
- # ------ -----
- # p2cwrite ---stdin---> p2cread
- # c2pread <--stdout--- c2pwrite
- # errread <--stderr--- errwrite
- #
- # On POSIX, the child objects are file descriptors. On
- # Windows, these are Windows file handles. The parent objects
- # are file descriptors on both platforms. The parent objects
- # are -1 when not using PIPEs. The child objects are -1
- # when not redirecting.
-
- (p2cread, p2cwrite,
- c2pread, c2pwrite,
- errread, errwrite) = self._get_handles(stdin, stdout, stderr)
-
- # We wrap OS handles *before* launching the child, otherwise a
- # quickly terminating child could make our fds unwrappable
- # (see #8458).
-
- if _mswindows:
- if p2cwrite != -1:
- p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
- if c2pread != -1:
- c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
- if errread != -1:
- errread = msvcrt.open_osfhandle(errread.Detach(), 0)
-
self.text_mode = encoding or errors or text or universal_newlines
-
- # PEP 597: We suppress the EncodingWarning in subprocess module
- # for now (at Python 3.10), because we focus on files for now.
- # This will be changed to encoding = io.text_encoding(encoding)
- # in the future.
if self.text_mode and encoding is None:
- self.encoding = encoding = "locale"
+ self.encoding = encoding = _text_encoding()
# How long to resume waiting on a child after the first ^C.
# There is no right value for this. The purpose is to be polite
@@ -873,6 +892,9 @@
else:
line_buffering = False
+ if process_group is None:
+ process_group = -1 # The internal APIs are int-only
+
gid = None
if group is not None:
if not hasattr(os, 'setregid'):
@@ -950,6 +972,39 @@
if uid < 0:
raise ValueError(f"User ID cannot be negative, got {uid}")
+ # Input and output objects. The general principle is like
+ # this:
+ #
+ # Parent Child
+ # ------ -----
+ # p2cwrite ---stdin---> p2cread
+ # c2pread <--stdout--- c2pwrite
+ # errread <--stderr--- errwrite
+ #
+ # On POSIX, the child objects are file descriptors. On
+ # Windows, these are Windows file handles. The parent objects
+ # are file descriptors on both platforms. The parent objects
+ # are -1 when not using PIPEs. The child objects are -1
+ # when not redirecting.
+
+ (p2cread, p2cwrite,
+ c2pread, c2pwrite,
+ errread, errwrite) = self._get_handles(stdin, stdout, stderr)
+
+ # From here on, raising exceptions may cause file descriptor leakage
+
+ # We wrap OS handles *before* launching the child, otherwise a
+ # quickly terminating child could make our fds unwrappable
+ # (see #8458).
+
+ if _mswindows:
+ if p2cwrite != -1:
+ p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
+ if c2pread != -1:
+ c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
+ if errread != -1:
+ errread = msvcrt.open_osfhandle(errread.Detach(), 0)
+
try:
if p2cwrite != -1:
self.stdin = io.open(p2cwrite, 'wb', bufsize)
@@ -976,7 +1031,7 @@
errread, errwrite,
restore_signals,
gid, gids, uid, umask,
- start_new_session)
+ start_new_session, process_group)
except:
# Cleanup if the child failed starting.
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
@@ -1253,6 +1308,26 @@
# Prevent a double close of these handles/fds from __init__ on error.
self._closed_child_pipe_fds = True
+ @contextlib.contextmanager
+ def _on_error_fd_closer(self):
+ """Helper to ensure file descriptors opened in _get_handles are closed"""
+ to_close = []
+ try:
+ yield to_close
+ except:
+ if hasattr(self, '_devnull'):
+ to_close.append(self._devnull)
+ del self._devnull
+ for fd in to_close:
+ try:
+ if _mswindows and isinstance(fd, Handle):
+ fd.Close()
+ else:
+ os.close(fd)
+ except OSError:
+ pass
+ raise
+
if _mswindows:
#
# Windows methods
@@ -1268,61 +1343,68 @@
c2pread, c2pwrite = -1, -1
errread, errwrite = -1, -1
- if stdin is None:
- p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
- if p2cread is None:
- p2cread, _ = _winapi.CreatePipe(None, 0)
- p2cread = Handle(p2cread)
- _winapi.CloseHandle(_)
- elif stdin == PIPE:
- p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
- p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
- elif stdin == DEVNULL:
- p2cread = msvcrt.get_osfhandle(self._get_devnull())
- elif isinstance(stdin, int):
- p2cread = msvcrt.get_osfhandle(stdin)
- else:
- # Assuming file-like object
- p2cread = msvcrt.get_osfhandle(stdin.fileno())
- p2cread = self._make_inheritable(p2cread)
+ with self._on_error_fd_closer() as err_close_fds:
+ if stdin is None:
+ p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
+ if p2cread is None:
+ p2cread, _ = _winapi.CreatePipe(None, 0)
+ p2cread = Handle(p2cread)
+ err_close_fds.append(p2cread)
+ _winapi.CloseHandle(_)
+ elif stdin == PIPE:
+ p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
+ p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
+ err_close_fds.extend((p2cread, p2cwrite))
+ elif stdin == DEVNULL:
+ p2cread = msvcrt.get_osfhandle(self._get_devnull())
+ elif isinstance(stdin, int):
+ p2cread = msvcrt.get_osfhandle(stdin)
+ else:
+ # Assuming file-like object
+ p2cread = msvcrt.get_osfhandle(stdin.fileno())
+ p2cread = self._make_inheritable(p2cread)
- if stdout is None:
- c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
- if c2pwrite is None:
- _, c2pwrite = _winapi.CreatePipe(None, 0)
- c2pwrite = Handle(c2pwrite)
- _winapi.CloseHandle(_)
- elif stdout == PIPE:
- c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
- c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
- elif stdout == DEVNULL:
- c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
- elif isinstance(stdout, int):
- c2pwrite = msvcrt.get_osfhandle(stdout)
- else:
- # Assuming file-like object
- c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
- c2pwrite = self._make_inheritable(c2pwrite)
+ if stdout is None:
+ c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
+ if c2pwrite is None:
+ _, c2pwrite = _winapi.CreatePipe(None, 0)
+ c2pwrite = Handle(c2pwrite)
+ err_close_fds.append(c2pwrite)
+ _winapi.CloseHandle(_)
+ elif stdout == PIPE:
+ c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
+ c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
+ err_close_fds.extend((c2pread, c2pwrite))
+ elif stdout == DEVNULL:
+ c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
+ elif isinstance(stdout, int):
+ c2pwrite = msvcrt.get_osfhandle(stdout)
+ else:
+ # Assuming file-like object
+ c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
+ c2pwrite = self._make_inheritable(c2pwrite)
- if stderr is None:
- errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
- if errwrite is None:
- _, errwrite = _winapi.CreatePipe(None, 0)
- errwrite = Handle(errwrite)
- _winapi.CloseHandle(_)
- elif stderr == PIPE:
- errread, errwrite = _winapi.CreatePipe(None, 0)
- errread, errwrite = Handle(errread), Handle(errwrite)
- elif stderr == STDOUT:
- errwrite = c2pwrite
- elif stderr == DEVNULL:
- errwrite = msvcrt.get_osfhandle(self._get_devnull())
- elif isinstance(stderr, int):
- errwrite = msvcrt.get_osfhandle(stderr)
- else:
- # Assuming file-like object
- errwrite = msvcrt.get_osfhandle(stderr.fileno())
- errwrite = self._make_inheritable(errwrite)
+ if stderr is None:
+ errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
+ if errwrite is None:
+ _, errwrite = _winapi.CreatePipe(None, 0)
+ errwrite = Handle(errwrite)
+ err_close_fds.append(errwrite)
+ _winapi.CloseHandle(_)
+ elif stderr == PIPE:
+ errread, errwrite = _winapi.CreatePipe(None, 0)
+ errread, errwrite = Handle(errread), Handle(errwrite)
+ err_close_fds.extend((errread, errwrite))
+ elif stderr == STDOUT:
+ errwrite = c2pwrite
+ elif stderr == DEVNULL:
+ errwrite = msvcrt.get_osfhandle(self._get_devnull())
+ elif isinstance(stderr, int):
+ errwrite = msvcrt.get_osfhandle(stderr)
+ else:
+ # Assuming file-like object
+ errwrite = msvcrt.get_osfhandle(stderr.fileno())
+ errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
@@ -1360,7 +1442,7 @@
unused_restore_signals,
unused_gid, unused_gids, unused_uid,
unused_umask,
- unused_start_new_session):
+ unused_start_new_session, unused_process_group):
"""Execute program (MS Windows version)"""
assert not pass_fds, "pass_fds not supported on Windows."
@@ -1609,52 +1691,56 @@
c2pread, c2pwrite = -1, -1
errread, errwrite = -1, -1
- if stdin is None:
- pass
- elif stdin == PIPE:
- p2cread, p2cwrite = os.pipe()
- if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
- fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
- elif stdin == DEVNULL:
- p2cread = self._get_devnull()
- elif isinstance(stdin, int):
- p2cread = stdin
- else:
- # Assuming file-like object
- p2cread = stdin.fileno()
+ with self._on_error_fd_closer() as err_close_fds:
+ if stdin is None:
+ pass
+ elif stdin == PIPE:
+ p2cread, p2cwrite = os.pipe()
+ err_close_fds.extend((p2cread, p2cwrite))
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
+ elif stdin == DEVNULL:
+ p2cread = self._get_devnull()
+ elif isinstance(stdin, int):
+ p2cread = stdin
+ else:
+ # Assuming file-like object
+ p2cread = stdin.fileno()
- if stdout is None:
- pass
- elif stdout == PIPE:
- c2pread, c2pwrite = os.pipe()
- if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
- fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
- elif stdout == DEVNULL:
- c2pwrite = self._get_devnull()
- elif isinstance(stdout, int):
- c2pwrite = stdout
- else:
- # Assuming file-like object
- c2pwrite = stdout.fileno()
+ if stdout is None:
+ pass
+ elif stdout == PIPE:
+ c2pread, c2pwrite = os.pipe()
+ err_close_fds.extend((c2pread, c2pwrite))
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
+ elif stdout == DEVNULL:
+ c2pwrite = self._get_devnull()
+ elif isinstance(stdout, int):
+ c2pwrite = stdout
+ else:
+ # Assuming file-like object
+ c2pwrite = stdout.fileno()
- if stderr is None:
- pass
- elif stderr == PIPE:
- errread, errwrite = os.pipe()
- if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
- fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
- elif stderr == STDOUT:
- if c2pwrite != -1:
- errwrite = c2pwrite
- else: # child's stdout is not set, use parent's stdout
- errwrite = sys.__stdout__.fileno()
- elif stderr == DEVNULL:
- errwrite = self._get_devnull()
- elif isinstance(stderr, int):
- errwrite = stderr
- else:
- # Assuming file-like object
- errwrite = stderr.fileno()
+ if stderr is None:
+ pass
+ elif stderr == PIPE:
+ errread, errwrite = os.pipe()
+ err_close_fds.extend((errread, errwrite))
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
+ elif stderr == STDOUT:
+ if c2pwrite != -1:
+ errwrite = c2pwrite
+ else: # child's stdout is not set, use parent's stdout
+ errwrite = sys.__stdout__.fileno()
+ elif stderr == DEVNULL:
+ errwrite = self._get_devnull()
+ elif isinstance(stderr, int):
+ errwrite = stderr
+ else:
+ # Assuming file-like object
+ errwrite = stderr.fileno()
return (p2cread, p2cwrite,
c2pread, c2pwrite,
@@ -1708,7 +1794,7 @@
errread, errwrite,
restore_signals,
gid, gids, uid, umask,
- start_new_session):
+ start_new_session, process_group):
"""Execute program (POSIX version)"""
if isinstance(args, (str, bytes)):
@@ -1744,6 +1830,7 @@
and (c2pwrite == -1 or c2pwrite > 2)
and (errwrite == -1 or errwrite > 2)
and not start_new_session
+ and process_group == -1
and gid is None
and gids is None
and uid is None
@@ -1793,7 +1880,7 @@
for dir in os.get_exec_path(env))
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
- self.pid = _posixsubprocess.fork_exec(
+ self.pid = _fork_exec(
args, executable_list,
close_fds, tuple(sorted(map(int, fds_to_keep))),
cwd, env_list,
@@ -1801,8 +1888,8 @@
errread, errwrite,
errpipe_read, errpipe_write,
restore_signals, start_new_session,
- gid, gids, uid, umask,
- preexec_fn)
+ process_group, gid, gids, uid, umask,
+ preexec_fn, _USE_VFORK)
self._child_created = True
finally:
# be sure the FD is closed no matter what
@@ -1865,19 +1952,19 @@
def _handle_exitstatus(self, sts,
- waitstatus_to_exitcode=os.waitstatus_to_exitcode,
- _WIFSTOPPED=os.WIFSTOPPED,
- _WSTOPSIG=os.WSTOPSIG):
+ _waitstatus_to_exitcode=_waitstatus_to_exitcode,
+ _WIFSTOPPED=_WIFSTOPPED,
+ _WSTOPSIG=_WSTOPSIG):
"""All callers to this function MUST hold self._waitpid_lock."""
# This method is called (indirectly) by __del__, so it cannot
# refer to anything outside of its local scope.
if _WIFSTOPPED(sts):
self.returncode = -_WSTOPSIG(sts)
else:
- self.returncode = waitstatus_to_exitcode(sts)
+ self.returncode = _waitstatus_to_exitcode(sts)
- def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
- _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
+ def _internal_poll(self, _deadstate=None, _waitpid=_waitpid,
+ _WNOHANG=_WNOHANG, _ECHILD=errno.ECHILD):
"""Check if child process has terminated. Returns returncode
attribute.
@@ -2108,7 +2195,7 @@
try:
os.kill(self.pid, sig)
except ProcessLookupError:
- # Supress the race condition error; bpo-40550.
+ # Suppress the race condition error; bpo-40550.
pass
def terminate(self):
diff --git a/common/py3-stdlib/sunau.py b/common/py3-stdlib/sunau.py
index 79750a9..94c42f1 100644
--- a/common/py3-stdlib/sunau.py
+++ b/common/py3-stdlib/sunau.py
@@ -104,6 +104,9 @@
"""
from collections import namedtuple
+import warnings
+
+warnings._deprecated(__name__, remove=(3, 13))
_sunau_params = namedtuple('_sunau_params',
@@ -275,7 +278,9 @@
data = self._file.read(nframes * self._framesize)
self._soundpos += len(data) // self._framesize
if self._encoding == AUDIO_FILE_ENCODING_MULAW_8:
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
data = audioop.ulaw2lin(data, self._sampwidth)
return data
return None # XXX--not implemented yet
@@ -421,7 +426,9 @@
data = memoryview(data).cast('B')
self._ensure_header_written()
if self._comptype == 'ULAW':
- import audioop
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ import audioop
data = audioop.lin2ulaw(data, self._sampwidth)
nframes = len(data) // self._framesize
self._file.write(data)
diff --git a/common/py3-stdlib/symtable.py b/common/py3-stdlib/symtable.py
index e11e5ff..5dd71ff 100644
--- a/common/py3-stdlib/symtable.py
+++ b/common/py3-stdlib/symtable.py
@@ -62,7 +62,7 @@
def get_type(self):
"""Return the type of the symbol table.
- The values retuned are 'class', 'module' and
+ The values returned are 'class', 'module' and
'function'.
"""
if self._table.type == _symtable.TYPE_MODULE:
@@ -245,7 +245,7 @@
return bool(self.__flags & DEF_PARAM)
def is_global(self):
- """Return *True* if the sysmbol is global.
+ """Return *True* if the symbol is global.
"""
return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
or (self.__module_scope and self.__flags & DEF_BOUND))
@@ -306,11 +306,15 @@
def get_namespace(self):
"""Return the single namespace bound to this name.
- Raises ValueError if the name is bound to multiple namespaces.
+ Raises ValueError if the name is bound to multiple namespaces
+ or no namespace.
"""
- if len(self.__namespaces) != 1:
+ if len(self.__namespaces) == 0:
+ raise ValueError("name is not bound to any namespaces")
+ elif len(self.__namespaces) > 1:
raise ValueError("name is bound to multiple namespaces")
- return self.__namespaces[0]
+ else:
+ return self.__namespaces[0]
if __name__ == "__main__":
import os, sys
diff --git a/common/py3-stdlib/sysconfig.py b/common/py3-stdlib/sysconfig.py
index daf9f00..ebe3711 100644
--- a/common/py3-stdlib/sysconfig.py
+++ b/common/py3-stdlib/sysconfig.py
@@ -56,8 +56,53 @@
'scripts': '{base}/Scripts',
'data': '{base}',
},
+ # Downstream distributors can overwrite the default install scheme.
+ # This is done to support downstream modifications where distributors change
+ # the installation layout (eg. different site-packages directory).
+ # So, distributors will change the default scheme to one that correctly
+ # represents their layout.
+ # This presents an issue for projects/people that need to bootstrap virtual
+ # environments, like virtualenv. As distributors might now be customizing
+ # the default install scheme, there is no guarantee that the information
+ # returned by sysconfig.get_default_scheme/get_paths is correct for
+ # a virtual environment, the only guarantee we have is that it is correct
+ # for the *current* environment. When bootstrapping a virtual environment,
+ # we need to know its layout, so that we can place the files in the
+ # correct locations.
+ # The "*_venv" install scheme is a scheme to bootstrap virtual environments,
+ # essentially identical to the default posix_prefix/nt schemes.
+ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to
+ # leave the following schemes unchanged
+ 'posix_venv': {
+ 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}',
+ 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}',
+ 'purelib': '{base}/lib/python{py_version_short}/site-packages',
+ 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages',
+ 'include':
+ '{installed_base}/include/python{py_version_short}{abiflags}',
+ 'platinclude':
+ '{installed_platbase}/include/python{py_version_short}{abiflags}',
+ 'scripts': '{base}/bin',
+ 'data': '{base}',
+ },
+ 'nt_venv': {
+ 'stdlib': '{installed_base}/Lib',
+ 'platstdlib': '{base}/Lib',
+ 'purelib': '{base}/Lib/site-packages',
+ 'platlib': '{base}/Lib/site-packages',
+ 'include': '{installed_base}/Include',
+ 'platinclude': '{installed_base}/Include',
+ 'scripts': '{base}/Scripts',
+ 'data': '{base}',
+ },
}
+# For the OS-native venv scheme, we essentially provide an alias:
+if os.name == 'nt':
+ _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['nt_venv']
+else:
+ _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv']
+
# NOTE: site.py has copy of this function.
# Sync it when modify this function.
@@ -66,8 +111,8 @@
if env_base:
return env_base
- # VxWorks has no home directories
- if sys.platform == "vxworks":
+ # Emscripten, VxWorks, and WASI have no home directories
+ if sys.platform in {"emscripten", "vxworks", "wasi"}:
return None
def joinuser(*args):
@@ -150,37 +195,38 @@
# unable to retrieve the real program name
_PROJECT_BASE = _safe_realpath(os.getcwd())
-if (os.name == 'nt' and
- _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
- _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
+# In a virtual environment, `sys._home` gives us the target directory
+# `_PROJECT_BASE` for the executable that created it when the virtual
+# python is an actual executable ('venv --copies' or Windows).
+_sys_home = getattr(sys, '_home', None)
+if _sys_home:
+ _PROJECT_BASE = _sys_home
+
+if os.name == 'nt':
+ # In a source build, the executable is in a subdirectory of the root
+ # that we want (<root>\PCbuild\<platname>).
+ # `_BASE_PREFIX` is used as the base installation is where the source
+ # will be. The realpath is needed to prevent mount point confusion
+ # that can occur with just string comparisons.
+ if _safe_realpath(_PROJECT_BASE).startswith(
+ _safe_realpath(f'{_BASE_PREFIX}\\PCbuild')):
+ _PROJECT_BASE = _BASE_PREFIX
# set for cross builds
if "_PYTHON_PROJECT_BASE" in os.environ:
_PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"])
-def _is_python_source_dir(d):
+def is_python_build(check_home=None):
+ if check_home is not None:
+ import warnings
+ warnings.warn("check_home argument is deprecated and ignored.",
+ DeprecationWarning, stacklevel=2)
for fn in ("Setup", "Setup.local"):
- if os.path.isfile(os.path.join(d, "Modules", fn)):
+ if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)):
return True
return False
-_sys_home = getattr(sys, '_home', None)
-
-if os.name == 'nt':
- def _fix_pcbuild(d):
- if d and os.path.normcase(d).startswith(
- os.path.normcase(os.path.join(_PREFIX, "PCbuild"))):
- return _PREFIX
- return d
- _PROJECT_BASE = _fix_pcbuild(_PROJECT_BASE)
- _sys_home = _fix_pcbuild(_sys_home)
-
-def is_python_build(check_home=False):
- if check_home and _sys_home:
- return _is_python_source_dir(_sys_home)
- return _is_python_source_dir(_PROJECT_BASE)
-
-_PYTHON_BUILD = is_python_build(True)
+_PYTHON_BUILD = is_python_build()
if _PYTHON_BUILD:
for scheme in ('posix_prefix', 'posix_home'):
@@ -192,6 +238,7 @@
scheme['headers'] = scheme['include']
scheme['include'] = '{srcdir}/Include'
scheme['platinclude'] = '{projectbase}/.'
+ del scheme
def _subst_vars(s, local_vars):
@@ -216,6 +263,11 @@
if vars is None:
vars = {}
_extend_dict(vars, get_config_vars())
+ if os.name == 'nt':
+ # On Windows we want to substitute 'lib' for schemes rather
+ # than the native value (without modifying vars, in case it
+ # was passed in)
+ vars = vars | {'platlibdir': 'lib'}
for key, value in _INSTALL_SCHEMES[scheme].items():
if os.name in ('posix', 'nt'):
@@ -245,6 +297,8 @@
def get_preferred_scheme(key):
+ if key == 'prefix' and sys.prefix != sys.base_prefix:
+ return 'venv'
scheme = _get_preferred_schemes()[key]
if scheme not in _INSTALL_SCHEMES:
raise ValueError(
@@ -389,7 +443,7 @@
def get_makefile_filename():
"""Return the path of the Makefile."""
if _PYTHON_BUILD:
- return os.path.join(_sys_home or _PROJECT_BASE, "Makefile")
+ return os.path.join(_PROJECT_BASE, "Makefile")
if hasattr(sys, 'abiflags'):
config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}'
else:
@@ -534,9 +588,9 @@
"""Return the path of pyconfig.h."""
if _PYTHON_BUILD:
if os.name == "nt":
- inc_dir = os.path.join(_sys_home or _PROJECT_BASE, "PC")
+ inc_dir = os.path.join(_PROJECT_BASE, "PC")
else:
- inc_dir = _sys_home or _PROJECT_BASE
+ inc_dir = _PROJECT_BASE
else:
inc_dir = get_path('platinclude')
return os.path.join(inc_dir, 'pyconfig.h')
@@ -611,12 +665,9 @@
if os.name == 'nt':
_init_non_posix(_CONFIG_VARS)
+ _CONFIG_VARS['VPATH'] = sys._vpath
if os.name == 'posix':
_init_posix(_CONFIG_VARS)
- # For backward compatibility, see issue19555
- SO = _CONFIG_VARS.get('EXT_SUFFIX')
- if SO is not None:
- _CONFIG_VARS['SO'] = SO
if _HAS_USER_BASE:
# Setting 'userbase' is done below the call to the
# init function to enable using 'get_config_var' in
@@ -661,9 +712,6 @@
Equivalent to get_config_vars().get(name)
"""
- if name == 'SO':
- import warnings
- warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2)
return get_config_vars().get(name)
diff --git a/common/py3-stdlib/tarfile.py b/common/py3-stdlib/tarfile.py
index dea150e..130b5e0 100755
--- a/common/py3-stdlib/tarfile.py
+++ b/common/py3-stdlib/tarfile.py
@@ -46,6 +46,7 @@
import struct
import copy
import re
+import warnings
try:
import pwd
@@ -71,6 +72,7 @@
"ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT",
"DEFAULT_FORMAT", "open"]
+
#---------------------------------------------------------
# tar constants
#---------------------------------------------------------
@@ -158,6 +160,8 @@
def stn(s, length, encoding, errors):
"""Convert a string to a null-terminated bytes object.
"""
+ if s is None:
+ raise ValueError("metadata cannot contain None")
s = s.encode(encoding, errors)
return s[:length] + (length - len(s)) * NUL
@@ -709,9 +713,127 @@
super().__init__(fileobj)
#class ExFileObject
+
+#-----------------------------
+# extraction filters (PEP 706)
+#-----------------------------
+
+class FilterError(TarError):
+ pass
+
+class AbsolutePathError(FilterError):
+ def __init__(self, tarinfo):
+ self.tarinfo = tarinfo
+ super().__init__(f'member {tarinfo.name!r} has an absolute path')
+
+class OutsideDestinationError(FilterError):
+ def __init__(self, tarinfo, path):
+ self.tarinfo = tarinfo
+ self._path = path
+ super().__init__(f'{tarinfo.name!r} would be extracted to {path!r}, '
+ + 'which is outside the destination')
+
+class SpecialFileError(FilterError):
+ def __init__(self, tarinfo):
+ self.tarinfo = tarinfo
+ super().__init__(f'{tarinfo.name!r} is a special file')
+
+class AbsoluteLinkError(FilterError):
+ def __init__(self, tarinfo):
+ self.tarinfo = tarinfo
+ super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path')
+
+class LinkOutsideDestinationError(FilterError):
+ def __init__(self, tarinfo, path):
+ self.tarinfo = tarinfo
+ self._path = path
+ super().__init__(f'{tarinfo.name!r} would link to {path!r}, '
+ + 'which is outside the destination')
+
+def _get_filtered_attrs(member, dest_path, for_data=True):
+ new_attrs = {}
+ name = member.name
+ dest_path = os.path.realpath(dest_path)
+ # Strip leading / (tar's directory separator) from filenames.
+ # Include os.sep (target OS directory separator) as well.
+ if name.startswith(('/', os.sep)):
+ name = new_attrs['name'] = member.path.lstrip('/' + os.sep)
+ if os.path.isabs(name):
+ # Path is absolute even after stripping.
+ # For example, 'C:/foo' on Windows.
+ raise AbsolutePathError(member)
+ # Ensure we stay in the destination
+ target_path = os.path.realpath(os.path.join(dest_path, name))
+ if os.path.commonpath([target_path, dest_path]) != dest_path:
+ raise OutsideDestinationError(member, target_path)
+ # Limit permissions (no high bits, and go-w)
+ mode = member.mode
+ if mode is not None:
+ # Strip high bits & group/other write bits
+ mode = mode & 0o755
+ if for_data:
+ # For data, handle permissions & file types
+ if member.isreg() or member.islnk():
+ if not mode & 0o100:
+ # Clear executable bits if not executable by user
+ mode &= ~0o111
+ # Ensure owner can read & write
+ mode |= 0o600
+ elif member.isdir() or member.issym():
+ # Ignore mode for directories & symlinks
+ mode = None
+ else:
+ # Reject special files
+ raise SpecialFileError(member)
+ if mode != member.mode:
+ new_attrs['mode'] = mode
+ if for_data:
+ # Ignore ownership for 'data'
+ if member.uid is not None:
+ new_attrs['uid'] = None
+ if member.gid is not None:
+ new_attrs['gid'] = None
+ if member.uname is not None:
+ new_attrs['uname'] = None
+ if member.gname is not None:
+ new_attrs['gname'] = None
+ # Check link destination for 'data'
+ if member.islnk() or member.issym():
+ if os.path.isabs(member.linkname):
+ raise AbsoluteLinkError(member)
+ target_path = os.path.realpath(os.path.join(dest_path, member.linkname))
+ if os.path.commonpath([target_path, dest_path]) != dest_path:
+ raise LinkOutsideDestinationError(member, target_path)
+ return new_attrs
+
+def fully_trusted_filter(member, dest_path):
+ return member
+
+def tar_filter(member, dest_path):
+ new_attrs = _get_filtered_attrs(member, dest_path, False)
+ if new_attrs:
+ return member.replace(**new_attrs, deep=False)
+ return member
+
+def data_filter(member, dest_path):
+ new_attrs = _get_filtered_attrs(member, dest_path, True)
+ if new_attrs:
+ return member.replace(**new_attrs, deep=False)
+ return member
+
+_NAMED_FILTERS = {
+ "fully_trusted": fully_trusted_filter,
+ "tar": tar_filter,
+ "data": data_filter,
+}
+
#------------------
# Exported Classes
#------------------
+
+# Sentinel for replace() defaults, meaning "don't change the attribute"
+_KEEP = object()
+
class TarInfo(object):
"""Informational class which holds the details about an
archive member given by a tar header block.
@@ -792,12 +914,44 @@
def __repr__(self):
return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
+ def replace(self, *,
+ name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP,
+ uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP,
+ deep=True, _KEEP=_KEEP):
+ """Return a deep copy of self with the given attributes replaced.
+ """
+ if deep:
+ result = copy.deepcopy(self)
+ else:
+ result = copy.copy(self)
+ if name is not _KEEP:
+ result.name = name
+ if mtime is not _KEEP:
+ result.mtime = mtime
+ if mode is not _KEEP:
+ result.mode = mode
+ if linkname is not _KEEP:
+ result.linkname = linkname
+ if uid is not _KEEP:
+ result.uid = uid
+ if gid is not _KEEP:
+ result.gid = gid
+ if uname is not _KEEP:
+ result.uname = uname
+ if gname is not _KEEP:
+ result.gname = gname
+ return result
+
def get_info(self):
"""Return the TarInfo's attributes as a dictionary.
"""
+ if self.mode is None:
+ mode = None
+ else:
+ mode = self.mode & 0o7777
info = {
"name": self.name,
- "mode": self.mode & 0o7777,
+ "mode": mode,
"uid": self.uid,
"gid": self.gid,
"size": self.size,
@@ -820,6 +974,9 @@
"""Return a tar header as a string of 512 byte blocks.
"""
info = self.get_info()
+ for name, value in info.items():
+ if value is None:
+ raise ValueError("%s may not be None" % name)
if format == USTAR_FORMAT:
return self.create_ustar_header(info, encoding, errors)
@@ -950,6 +1107,12 @@
devmajor = stn("", 8, encoding, errors)
devminor = stn("", 8, encoding, errors)
+ # None values in metadata should cause ValueError.
+ # itn()/stn() do this for all fields except type.
+ filetype = info.get("type", REGTYPE)
+ if filetype is None:
+ raise ValueError("TarInfo.type must not be None")
+
parts = [
stn(info.get("name", ""), 100, encoding, errors),
itn(info.get("mode", 0) & 0o7777, 8, format),
@@ -958,7 +1121,7 @@
itn(info.get("size", 0), 12, format),
itn(info.get("mtime", 0), 12, format),
b" ", # checksum field
- info.get("type", REGTYPE),
+ filetype,
stn(info.get("linkname", ""), 100, encoding, errors),
info.get("magic", POSIX_MAGIC),
stn(info.get("uname", ""), 32, encoding, errors),
@@ -1468,6 +1631,8 @@
fileobject = ExFileObject # The file-object for extractfile().
+ extraction_filter = None # The default filter for extraction.
+
def __init__(self, name=None, mode="r", fileobj=None, format=None,
tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
errors="surrogateescape", pax_headers=None, debug=None,
@@ -1940,7 +2105,10 @@
members = self
for tarinfo in members:
if verbose:
- _safe_print(stat.filemode(tarinfo.mode))
+ if tarinfo.mode is None:
+ _safe_print("??????????")
+ else:
+ _safe_print(stat.filemode(tarinfo.mode))
_safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid,
tarinfo.gname or tarinfo.gid))
if tarinfo.ischr() or tarinfo.isblk():
@@ -1948,8 +2116,11 @@
("%d,%d" % (tarinfo.devmajor, tarinfo.devminor)))
else:
_safe_print("%10d" % tarinfo.size)
- _safe_print("%d-%02d-%02d %02d:%02d:%02d" \
- % time.localtime(tarinfo.mtime)[:6])
+ if tarinfo.mtime is None:
+ _safe_print("????-??-?? ??:??:??")
+ else:
+ _safe_print("%d-%02d-%02d %02d:%02d:%02d" \
+ % time.localtime(tarinfo.mtime)[:6])
_safe_print(tarinfo.name + ("/" if tarinfo.isdir() else ""))
@@ -2036,32 +2207,58 @@
self.members.append(tarinfo)
- def extractall(self, path=".", members=None, *, numeric_owner=False):
+ def _get_filter_function(self, filter):
+ if filter is None:
+ filter = self.extraction_filter
+ if filter is None:
+ return fully_trusted_filter
+ if isinstance(filter, str):
+ raise TypeError(
+ 'String names are not supported for '
+ + 'TarFile.extraction_filter. Use a function such as '
+ + 'tarfile.data_filter directly.')
+ return filter
+ if callable(filter):
+ return filter
+ try:
+ return _NAMED_FILTERS[filter]
+ except KeyError:
+ raise ValueError(f"filter {filter!r} not found") from None
+
+ def extractall(self, path=".", members=None, *, numeric_owner=False,
+ filter=None):
"""Extract all members from the archive to the current working
directory and set owner, modification time and permissions on
directories afterwards. `path' specifies a different directory
to extract to. `members' is optional and must be a subset of the
list returned by getmembers(). If `numeric_owner` is True, only
the numbers for user/group names are used and not the names.
+
+ The `filter` function will be called on each member just
+ before extraction.
+ It can return a changed TarInfo or None to skip the member.
+ String names of common filters are accepted.
"""
directories = []
+ filter_function = self._get_filter_function(filter)
if members is None:
members = self
- for tarinfo in members:
+ for member in members:
+ tarinfo = self._get_extract_tarinfo(member, filter_function, path)
+ if tarinfo is None:
+ continue
if tarinfo.isdir():
- # Extract directories with a safe mode.
+ # For directories, delay setting attributes until later,
+ # since permissions can interfere with extraction and
+ # extracting contents can reset mtime.
directories.append(tarinfo)
- tarinfo = copy.copy(tarinfo)
- tarinfo.mode = 0o700
- # Do not set_attrs directories, as we will do that further down
- self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(),
- numeric_owner=numeric_owner)
+ self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(),
+ numeric_owner=numeric_owner)
# Reverse sort directories.
- directories.sort(key=lambda a: a.name)
- directories.reverse()
+ directories.sort(key=lambda a: a.name, reverse=True)
# Set correct owner, mtime and filemode on directories.
for tarinfo in directories:
@@ -2071,12 +2268,10 @@
self.utime(tarinfo, dirpath)
self.chmod(tarinfo, dirpath)
except ExtractError as e:
- if self.errorlevel > 1:
- raise
- else:
- self._dbg(1, "tarfile: %s" % e)
+ self._handle_nonfatal_error(e)
- def extract(self, member, path="", set_attrs=True, *, numeric_owner=False):
+ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False,
+ filter=None):
"""Extract a member from the archive to the current working directory,
using its full name. Its file information is extracted as accurately
as possible. `member' may be a filename or a TarInfo object. You can
@@ -2084,35 +2279,70 @@
mtime, mode) are set unless `set_attrs' is False. If `numeric_owner`
is True, only the numbers for user/group names are used and not
the names.
- """
- self._check("r")
+ The `filter` function will be called before extraction.
+ It can return a changed TarInfo or None to skip the member.
+ String names of common filters are accepted.
+ """
+ filter_function = self._get_filter_function(filter)
+ tarinfo = self._get_extract_tarinfo(member, filter_function, path)
+ if tarinfo is not None:
+ self._extract_one(tarinfo, path, set_attrs, numeric_owner)
+
+ def _get_extract_tarinfo(self, member, filter_function, path):
+ """Get filtered TarInfo (or None) from member, which might be a str"""
if isinstance(member, str):
tarinfo = self.getmember(member)
else:
tarinfo = member
+ unfiltered = tarinfo
+ try:
+ tarinfo = filter_function(tarinfo, path)
+ except (OSError, FilterError) as e:
+ self._handle_fatal_error(e)
+ except ExtractError as e:
+ self._handle_nonfatal_error(e)
+ if tarinfo is None:
+ self._dbg(2, "tarfile: Excluded %r" % unfiltered.name)
+ return None
# Prepare the link target for makelink().
if tarinfo.islnk():
+ tarinfo = copy.copy(tarinfo)
tarinfo._link_target = os.path.join(path, tarinfo.linkname)
+ return tarinfo
+
+ def _extract_one(self, tarinfo, path, set_attrs, numeric_owner):
+ """Extract from filtered tarinfo to disk"""
+ self._check("r")
try:
self._extract_member(tarinfo, os.path.join(path, tarinfo.name),
set_attrs=set_attrs,
numeric_owner=numeric_owner)
except OSError as e:
- if self.errorlevel > 0:
- raise
- else:
- if e.filename is None:
- self._dbg(1, "tarfile: %s" % e.strerror)
- else:
- self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
+ self._handle_fatal_error(e)
except ExtractError as e:
- if self.errorlevel > 1:
- raise
+ self._handle_nonfatal_error(e)
+
+ def _handle_nonfatal_error(self, e):
+ """Handle non-fatal error (ExtractError) according to errorlevel"""
+ if self.errorlevel > 1:
+ raise
+ else:
+ self._dbg(1, "tarfile: %s" % e)
+
+ def _handle_fatal_error(self, e):
+ """Handle "fatal" error according to self.errorlevel"""
+ if self.errorlevel > 0:
+ raise
+ elif isinstance(e, OSError):
+ if e.filename is None:
+ self._dbg(1, "tarfile: %s" % e.strerror)
else:
- self._dbg(1, "tarfile: %s" % e)
+ self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
+ else:
+ self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e))
def extractfile(self, member):
"""Extract a member from the archive as a file object. `member' may be
@@ -2199,9 +2429,13 @@
"""Make a directory called targetpath.
"""
try:
- # Use a safe mode for the directory, the real mode is set
- # later in _extract_member().
- os.mkdir(targetpath, 0o700)
+ if tarinfo.mode is None:
+ # Use the system's default mode
+ os.mkdir(targetpath)
+ else:
+ # Use a safe mode for the directory, the real mode is set
+ # later in _extract_member().
+ os.mkdir(targetpath, 0o700)
except FileExistsError:
pass
@@ -2244,6 +2478,9 @@
raise ExtractError("special devices not supported by system")
mode = tarinfo.mode
+ if mode is None:
+ # Use mknod's default
+ mode = 0o600
if tarinfo.isblk():
mode |= stat.S_IFBLK
else:
@@ -2265,7 +2502,6 @@
os.unlink(targetpath)
os.symlink(tarinfo.linkname, targetpath)
else:
- # See extract().
if os.path.exists(tarinfo._link_target):
os.link(tarinfo._link_target, targetpath)
else:
@@ -2290,15 +2526,19 @@
u = tarinfo.uid
if not numeric_owner:
try:
- if grp:
+ if grp and tarinfo.gname:
g = grp.getgrnam(tarinfo.gname)[2]
except KeyError:
pass
try:
- if pwd:
+ if pwd and tarinfo.uname:
u = pwd.getpwnam(tarinfo.uname)[2]
except KeyError:
pass
+ if g is None:
+ g = -1
+ if u is None:
+ u = -1
try:
if tarinfo.issym() and hasattr(os, "lchown"):
os.lchown(targetpath, u, g)
@@ -2310,6 +2550,8 @@
def chmod(self, tarinfo, targetpath):
"""Set file permissions of targetpath according to tarinfo.
"""
+ if tarinfo.mode is None:
+ return
try:
os.chmod(targetpath, tarinfo.mode)
except OSError as e:
@@ -2318,10 +2560,13 @@
def utime(self, tarinfo, targetpath):
"""Set modification time of targetpath according to tarinfo.
"""
+ mtime = tarinfo.mtime
+ if mtime is None:
+ return
if not hasattr(os, 'utime'):
return
try:
- os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
+ os.utime(targetpath, (mtime, mtime))
except OSError as e:
raise ExtractError("could not change modification time") from e
@@ -2339,6 +2584,8 @@
# Advance the file pointer.
if self.offset != self.fileobj.tell():
+ if self.offset == 0:
+ return None
self.fileobj.seek(self.offset - 1)
if not self.fileobj.read(1):
raise ReadError("unexpected end of data")
@@ -2397,13 +2644,26 @@
members = self.getmembers()
# Limit the member search list up to tarinfo.
+ skipping = False
if tarinfo is not None:
- members = members[:members.index(tarinfo)]
+ try:
+ index = members.index(tarinfo)
+ except ValueError:
+ # The given starting point might be a (modified) copy.
+ # We'll later skip members until we find an equivalent.
+ skipping = True
+ else:
+ # Happy fast path
+ members = members[:index]
if normalize:
name = os.path.normpath(name)
for member in reversed(members):
+ if skipping:
+ if tarinfo.offset == member.offset:
+ skipping = False
+ continue
if normalize:
member_name = os.path.normpath(member.name)
else:
@@ -2412,6 +2672,10 @@
if name == member_name:
return member
+ if skipping:
+ # Starting point was not found
+ raise ValueError(tarinfo)
+
def _load(self):
"""Read through the entire archive file and look for readable
members.
@@ -2504,6 +2768,7 @@
#--------------------
# exported functions
#--------------------
+
def is_tarfile(name):
"""Return True if name points to a tar archive that we
are able to handle, else return False.
@@ -2512,7 +2777,9 @@
"""
try:
if hasattr(name, "read"):
+ pos = name.tell()
t = open(fileobj=name)
+ name.seek(pos)
else:
t = open(name)
t.close()
@@ -2530,6 +2797,10 @@
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-v', '--verbose', action='store_true', default=False,
help='Verbose output')
+ parser.add_argument('--filter', metavar='<filtername>',
+ choices=_NAMED_FILTERS,
+ help='Filter for extraction')
+
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-l', '--list', metavar='<tarfile>',
help='Show listing of a tarfile')
@@ -2541,8 +2812,12 @@
help='Create tarfile from sources')
group.add_argument('-t', '--test', metavar='<tarfile>',
help='Test if a tarfile is valid')
+
args = parser.parse_args()
+ if args.filter and args.extract is None:
+ parser.exit(1, '--filter is only valid for extraction\n')
+
if args.test is not None:
src = args.test
if is_tarfile(src):
@@ -2573,7 +2848,7 @@
if is_tarfile(src):
with TarFile.open(src, 'r:*') as tf:
- tf.extractall(path=curdir)
+ tf.extractall(path=curdir, filter=args.filter)
if args.verbose:
if curdir == '.':
msg = '{!r} file is extracted.'.format(src)
diff --git a/common/py3-stdlib/telnetlib.py b/common/py3-stdlib/telnetlib.py
index 8ce053e..62d6361 100644
--- a/common/py3-stdlib/telnetlib.py
+++ b/common/py3-stdlib/telnetlib.py
@@ -37,6 +37,9 @@
import socket
import selectors
from time import monotonic as _time
+import warnings
+
+warnings._deprecated(__name__, remove=(3, 13))
__all__ = ["Telnet"]
@@ -489,7 +492,6 @@
except EOFError: # raised by self.rawq_getchar()
self.iacseq = b'' # Reset on EOF
self.sb = 0
- pass
self.cookedq = self.cookedq + buf[0]
self.sbdataq = self.sbdataq + buf[1]
diff --git a/common/py3-stdlib/tempfile.py b/common/py3-stdlib/tempfile.py
index 96da930..480c172 100644
--- a/common/py3-stdlib/tempfile.py
+++ b/common/py3-stdlib/tempfile.py
@@ -536,6 +536,10 @@
Returns an object with a file-like interface; the name of the file
is accessible as its 'name' attribute. The file will be automatically
deleted when it is closed unless the 'delete' argument is set to False.
+
+ On POSIX, NamedTemporaryFiles cannot be automatically deleted if
+ the creating process is terminated abruptly with a SIGKILL signal.
+ Windows can delete the file even in this case.
"""
prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir)
@@ -658,7 +662,7 @@
raw.name = fd
return file
-class SpooledTemporaryFile:
+class SpooledTemporaryFile(_io.IOBase):
"""Temporary file wrapper, specialized to switch from BytesIO
or StringIO to a real file when it exceeds a certain size or
when a fileno is needed.
@@ -723,6 +727,16 @@
def __iter__(self):
return self._file.__iter__()
+ def __del__(self):
+ if not self.closed:
+ _warnings.warn(
+ "Unclosed file {!r}".format(self),
+ ResourceWarning,
+ stacklevel=2,
+ source=self
+ )
+ self.close()
+
def close(self):
self._file.close()
@@ -766,15 +780,30 @@
def newlines(self):
return self._file.newlines
+ def readable(self):
+ return self._file.readable()
+
def read(self, *args):
return self._file.read(*args)
+ def read1(self, *args):
+ return self._file.read1(*args)
+
+ def readinto(self, b):
+ return self._file.readinto(b)
+
+ def readinto1(self, b):
+ return self._file.readinto1(b)
+
def readline(self, *args):
return self._file.readline(*args)
def readlines(self, *args):
return self._file.readlines(*args)
+ def seekable(self):
+ return self._file.seekable()
+
def seek(self, *args):
return self._file.seek(*args)
@@ -783,11 +812,14 @@
def truncate(self, size=None):
if size is None:
- self._file.truncate()
+ return self._file.truncate()
else:
if size > self._max_size:
self.rollover()
- self._file.truncate(size)
+ return self._file.truncate(size)
+
+ def writable(self):
+ return self._file.writable()
def write(self, s):
file = self._file
@@ -801,6 +833,9 @@
self._check(file)
return rv
+ def detach(self):
+ return self._file.detach()
+
class TemporaryDirectory:
"""Create and return a temporary directory. This has the same
diff --git a/common/py3-stdlib/textwrap.py b/common/py3-stdlib/textwrap.py
index 841de9b..98bedd2 100644
--- a/common/py3-stdlib/textwrap.py
+++ b/common/py3-stdlib/textwrap.py
@@ -63,10 +63,7 @@
Append to the last line of truncated text.
"""
- unicode_whitespace_trans = {}
- uspace = ord(' ')
- for x in _whitespace:
- unicode_whitespace_trans[ord(x)] = uspace
+ unicode_whitespace_trans = dict.fromkeys(map(ord, _whitespace), ord(' '))
# This funky little regex is just the trick for splitting
# text up into word-wrappable chunks. E.g.
diff --git a/common/py3-stdlib/threading.py b/common/py3-stdlib/threading.py
index 62f49c0..4f72938 100644
--- a/common/py3-stdlib/threading.py
+++ b/common/py3-stdlib/threading.py
@@ -425,6 +425,11 @@
self._cond = Condition(Lock())
self._value = value
+ def __repr__(self):
+ cls = self.__class__
+ return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:"
+ f" value={self._value}>")
+
def acquire(self, blocking=True, timeout=None):
"""Acquire a semaphore, decrementing the internal counter by one.
@@ -511,6 +516,11 @@
Semaphore.__init__(self, value)
self._initial_value = value
+ def __repr__(self):
+ cls = self.__class__
+ return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:"
+ f" value={self._value}/{self._initial_value}>")
+
def release(self, n=1):
"""Release a semaphore, incrementing the internal counter by one or more.
@@ -546,6 +556,11 @@
self._cond = Condition(Lock())
self._flag = False
+ def __repr__(self):
+ cls = self.__class__
+ status = 'set' if self._flag else 'unset'
+ return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: {status}>"
+
def _at_fork_reinit(self):
# Private method called by Thread._reset_internal_locks()
self._cond._at_fork_reinit()
@@ -644,6 +659,13 @@
self._state = 0 # 0 filling, 1 draining, -1 resetting, -2 broken
self._count = 0
+ def __repr__(self):
+ cls = self.__class__
+ if self.broken:
+ return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: broken>"
+ return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:"
+ f" waiters={self.n_waiting}/{self.parties}>")
+
def wait(self, timeout=None):
"""Wait for the barrier.
@@ -837,7 +859,7 @@
*name* is the thread name. By default, a unique name is constructed of
the form "Thread-N" where N is a small decimal number.
- *args* is the argument tuple for the target invocation. Defaults to ().
+ *args* is a list or tuple of arguments for the target invocation. Defaults to ().
*kwargs* is a dictionary of keyword arguments for the target
invocation. Defaults to {}.
@@ -1017,13 +1039,7 @@
except:
self._invoke_excepthook(self)
finally:
- with _active_limbo_lock:
- try:
- # We don't call self._delete() because it also
- # grabs _active_limbo_lock.
- del _active[get_ident()]
- except:
- pass
+ self._delete()
def _stop(self):
# After calling ._stop(), .is_alive() returns False and .join() returns
diff --git a/common/py3-stdlib/tokenize.py b/common/py3-stdlib/tokenize.py
index 7d7736f..46d2224 100644
--- a/common/py3-stdlib/tokenize.py
+++ b/common/py3-stdlib/tokenize.py
@@ -143,6 +143,7 @@
endpats[_prefix + '"'] = Double
endpats[_prefix + "'''"] = Single3
endpats[_prefix + '"""'] = Double3
+del _prefix
# A set of all of the single and triple quoted string prefixes,
# including the opening quotes.
@@ -153,6 +154,7 @@
single_quoted.add(u)
for u in (t + '"""', t + "'''"):
triple_quoted.add(u)
+del t, u
tabsize = 8
@@ -680,5 +682,13 @@
perror("unexpected error: %s" % err)
raise
+def _generate_tokens_from_c_tokenizer(source):
+ """Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
+ import _tokenize as c_tokenizer
+ for info in c_tokenizer.TokenizerIter(source):
+ tok, type, lineno, end_lineno, col_off, end_col_off, line = info
+ yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)
+
+
if __name__ == "__main__":
main()
diff --git a/common/py3-stdlib/tomllib/__init__.py b/common/py3-stdlib/tomllib/__init__.py
new file mode 100644
index 0000000..ef91cb9
--- /dev/null
+++ b/common/py3-stdlib/tomllib/__init__.py
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+# Licensed to PSF under a Contributor Agreement.
+
+__all__ = ("loads", "load", "TOMLDecodeError")
+
+from ._parser import TOMLDecodeError, load, loads
+
+# Pretend this exception was created here.
+TOMLDecodeError.__module__ = __name__
diff --git a/common/py3-stdlib/tomllib/_parser.py b/common/py3-stdlib/tomllib/_parser.py
new file mode 100644
index 0000000..45ca7a8
--- /dev/null
+++ b/common/py3-stdlib/tomllib/_parser.py
@@ -0,0 +1,691 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+# Licensed to PSF under a Contributor Agreement.
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+import string
+from types import MappingProxyType
+from typing import Any, BinaryIO, NamedTuple
+
+from ._re import (
+ RE_DATETIME,
+ RE_LOCALTIME,
+ RE_NUMBER,
+ match_to_datetime,
+ match_to_localtime,
+ match_to_number,
+)
+from ._types import Key, ParseFloat, Pos
+
+ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
+
+# Neither of these sets include quotation mark or backslash. They are
+# currently handled as separate cases in the parser functions.
+ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t")
+ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n")
+
+ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS
+ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ILLEGAL_MULTILINE_BASIC_STR_CHARS
+
+ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS
+
+TOML_WS = frozenset(" \t")
+TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n")
+BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_")
+KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'")
+HEXDIGIT_CHARS = frozenset(string.hexdigits)
+
+BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType(
+ {
+ "\\b": "\u0008", # backspace
+ "\\t": "\u0009", # tab
+ "\\n": "\u000A", # linefeed
+ "\\f": "\u000C", # form feed
+ "\\r": "\u000D", # carriage return
+ '\\"': "\u0022", # quote
+ "\\\\": "\u005C", # backslash
+ }
+)
+
+
+class TOMLDecodeError(ValueError):
+ """An error raised if a document is not valid TOML."""
+
+
+def load(fp: BinaryIO, /, *, parse_float: ParseFloat = float) -> dict[str, Any]:
+ """Parse TOML from a binary file object."""
+ b = fp.read()
+ try:
+ s = b.decode()
+ except AttributeError:
+ raise TypeError(
+ "File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`"
+ ) from None
+ return loads(s, parse_float=parse_float)
+
+
+def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # noqa: C901
+ """Parse TOML from a string."""
+
+ # The spec allows converting "\r\n" to "\n", even in string
+ # literals. Let's do so to simplify parsing.
+ src = s.replace("\r\n", "\n")
+ pos = 0
+ out = Output(NestedDict(), Flags())
+ header: Key = ()
+ parse_float = make_safe_parse_float(parse_float)
+
+ # Parse one statement at a time
+ # (typically means one line in TOML source)
+ while True:
+ # 1. Skip line leading whitespace
+ pos = skip_chars(src, pos, TOML_WS)
+
+ # 2. Parse rules. Expect one of the following:
+ # - end of file
+ # - end of line
+ # - comment
+ # - key/value pair
+ # - append dict to list (and move to its namespace)
+ # - create dict (and move to its namespace)
+ # Skip trailing whitespace when applicable.
+ try:
+ char = src[pos]
+ except IndexError:
+ break
+ if char == "\n":
+ pos += 1
+ continue
+ if char in KEY_INITIAL_CHARS:
+ pos = key_value_rule(src, pos, out, header, parse_float)
+ pos = skip_chars(src, pos, TOML_WS)
+ elif char == "[":
+ try:
+ second_char: str | None = src[pos + 1]
+ except IndexError:
+ second_char = None
+ out.flags.finalize_pending()
+ if second_char == "[":
+ pos, header = create_list_rule(src, pos, out)
+ else:
+ pos, header = create_dict_rule(src, pos, out)
+ pos = skip_chars(src, pos, TOML_WS)
+ elif char != "#":
+ raise suffixed_err(src, pos, "Invalid statement")
+
+ # 3. Skip comment
+ pos = skip_comment(src, pos)
+
+ # 4. Expect end of line or end of file
+ try:
+ char = src[pos]
+ except IndexError:
+ break
+ if char != "\n":
+ raise suffixed_err(
+ src, pos, "Expected newline or end of document after a statement"
+ )
+ pos += 1
+
+ return out.data.dict
+
+
+class Flags:
+ """Flags that map to parsed keys/namespaces."""
+
+ # Marks an immutable namespace (inline array or inline table).
+ FROZEN = 0
+ # Marks a nest that has been explicitly created and can no longer
+ # be opened using the "[table]" syntax.
+ EXPLICIT_NEST = 1
+
+ def __init__(self) -> None:
+ self._flags: dict[str, dict] = {}
+ self._pending_flags: set[tuple[Key, int]] = set()
+
+ def add_pending(self, key: Key, flag: int) -> None:
+ self._pending_flags.add((key, flag))
+
+ def finalize_pending(self) -> None:
+ for key, flag in self._pending_flags:
+ self.set(key, flag, recursive=False)
+ self._pending_flags.clear()
+
+ def unset_all(self, key: Key) -> None:
+ cont = self._flags
+ for k in key[:-1]:
+ if k not in cont:
+ return
+ cont = cont[k]["nested"]
+ cont.pop(key[-1], None)
+
+ def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003
+ cont = self._flags
+ key_parent, key_stem = key[:-1], key[-1]
+ for k in key_parent:
+ if k not in cont:
+ cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}}
+ cont = cont[k]["nested"]
+ if key_stem not in cont:
+ cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}}
+ cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag)
+
+ def is_(self, key: Key, flag: int) -> bool:
+ if not key:
+ return False # document root has no flags
+ cont = self._flags
+ for k in key[:-1]:
+ if k not in cont:
+ return False
+ inner_cont = cont[k]
+ if flag in inner_cont["recursive_flags"]:
+ return True
+ cont = inner_cont["nested"]
+ key_stem = key[-1]
+ if key_stem in cont:
+ cont = cont[key_stem]
+ return flag in cont["flags"] or flag in cont["recursive_flags"]
+ return False
+
+
+class NestedDict:
+ def __init__(self) -> None:
+ # The parsed content of the TOML document
+ self.dict: dict[str, Any] = {}
+
+ def get_or_create_nest(
+ self,
+ key: Key,
+ *,
+ access_lists: bool = True,
+ ) -> dict:
+ cont: Any = self.dict
+ for k in key:
+ if k not in cont:
+ cont[k] = {}
+ cont = cont[k]
+ if access_lists and isinstance(cont, list):
+ cont = cont[-1]
+ if not isinstance(cont, dict):
+ raise KeyError("There is no nest behind this key")
+ return cont
+
+ def append_nest_to_list(self, key: Key) -> None:
+ cont = self.get_or_create_nest(key[:-1])
+ last_key = key[-1]
+ if last_key in cont:
+ list_ = cont[last_key]
+ if not isinstance(list_, list):
+ raise KeyError("An object other than list found behind this key")
+ list_.append({})
+ else:
+ cont[last_key] = [{}]
+
+
+class Output(NamedTuple):
+ data: NestedDict
+ flags: Flags
+
+
+def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos:
+ try:
+ while src[pos] in chars:
+ pos += 1
+ except IndexError:
+ pass
+ return pos
+
+
+def skip_until(
+ src: str,
+ pos: Pos,
+ expect: str,
+ *,
+ error_on: frozenset[str],
+ error_on_eof: bool,
+) -> Pos:
+ try:
+ new_pos = src.index(expect, pos)
+ except ValueError:
+ new_pos = len(src)
+ if error_on_eof:
+ raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None
+
+ if not error_on.isdisjoint(src[pos:new_pos]):
+ while src[pos] not in error_on:
+ pos += 1
+ raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}")
+ return new_pos
+
+
+def skip_comment(src: str, pos: Pos) -> Pos:
+ try:
+ char: str | None = src[pos]
+ except IndexError:
+ char = None
+ if char == "#":
+ return skip_until(
+ src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False
+ )
+ return pos
+
+
+def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos:
+ while True:
+ pos_before_skip = pos
+ pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
+ pos = skip_comment(src, pos)
+ if pos == pos_before_skip:
+ return pos
+
+
+def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]:
+ pos += 1 # Skip "["
+ pos = skip_chars(src, pos, TOML_WS)
+ pos, key = parse_key(src, pos)
+
+ if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN):
+ raise suffixed_err(src, pos, f"Cannot declare {key} twice")
+ out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
+ try:
+ out.data.get_or_create_nest(key)
+ except KeyError:
+ raise suffixed_err(src, pos, "Cannot overwrite a value") from None
+
+ if not src.startswith("]", pos):
+ raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
+ return pos + 1, key
+
+
+def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]:
+ pos += 2 # Skip "[["
+ pos = skip_chars(src, pos, TOML_WS)
+ pos, key = parse_key(src, pos)
+
+ if out.flags.is_(key, Flags.FROZEN):
+ raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}")
+ # Free the namespace now that it points to another empty list item...
+ out.flags.unset_all(key)
+ # ...but this key precisely is still prohibited from table declaration
+ out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
+ try:
+ out.data.append_nest_to_list(key)
+ except KeyError:
+ raise suffixed_err(src, pos, "Cannot overwrite a value") from None
+
+ if not src.startswith("]]", pos):
+ raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
+ return pos + 2, key
+
+
+def key_value_rule(
+ src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat
+) -> Pos:
+ pos, key, value = parse_key_value_pair(src, pos, parse_float)
+ key_parent, key_stem = key[:-1], key[-1]
+ abs_key_parent = header + key_parent
+
+ relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
+ for cont_key in relative_path_cont_keys:
+ # Check that dotted key syntax does not redefine an existing table
+ if out.flags.is_(cont_key, Flags.EXPLICIT_NEST):
+ raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}")
+ # Containers in the relative path can't be opened with the table syntax or
+ # dotted key/value syntax in following table sections.
+ out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST)
+
+ if out.flags.is_(abs_key_parent, Flags.FROZEN):
+ raise suffixed_err(
+ src, pos, f"Cannot mutate immutable namespace {abs_key_parent}"
+ )
+
+ try:
+ nest = out.data.get_or_create_nest(abs_key_parent)
+ except KeyError:
+ raise suffixed_err(src, pos, "Cannot overwrite a value") from None
+ if key_stem in nest:
+ raise suffixed_err(src, pos, "Cannot overwrite a value")
+ # Mark inline table and array namespaces recursively immutable
+ if isinstance(value, (dict, list)):
+ out.flags.set(header + key, Flags.FROZEN, recursive=True)
+ nest[key_stem] = value
+ return pos
+
+
+def parse_key_value_pair(
+ src: str, pos: Pos, parse_float: ParseFloat
+) -> tuple[Pos, Key, Any]:
+ pos, key = parse_key(src, pos)
+ try:
+ char: str | None = src[pos]
+ except IndexError:
+ char = None
+ if char != "=":
+ raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
+ pos += 1
+ pos = skip_chars(src, pos, TOML_WS)
+ pos, value = parse_value(src, pos, parse_float)
+ return pos, key, value
+
+
+def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]:
+ pos, key_part = parse_key_part(src, pos)
+ key: Key = (key_part,)
+ pos = skip_chars(src, pos, TOML_WS)
+ while True:
+ try:
+ char: str | None = src[pos]
+ except IndexError:
+ char = None
+ if char != ".":
+ return pos, key
+ pos += 1
+ pos = skip_chars(src, pos, TOML_WS)
+ pos, key_part = parse_key_part(src, pos)
+ key += (key_part,)
+ pos = skip_chars(src, pos, TOML_WS)
+
+
+def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]:
+ try:
+ char: str | None = src[pos]
+ except IndexError:
+ char = None
+ if char in BARE_KEY_CHARS:
+ start_pos = pos
+ pos = skip_chars(src, pos, BARE_KEY_CHARS)
+ return pos, src[start_pos:pos]
+ if char == "'":
+ return parse_literal_str(src, pos)
+ if char == '"':
+ return parse_one_line_basic_str(src, pos)
+ raise suffixed_err(src, pos, "Invalid initial character for a key part")
+
+
+def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]:
+ pos += 1
+ return parse_basic_str(src, pos, multiline=False)
+
+
+def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list]:
+ pos += 1
+ array: list = []
+
+ pos = skip_comments_and_array_ws(src, pos)
+ if src.startswith("]", pos):
+ return pos + 1, array
+ while True:
+ pos, val = parse_value(src, pos, parse_float)
+ array.append(val)
+ pos = skip_comments_and_array_ws(src, pos)
+
+ c = src[pos : pos + 1]
+ if c == "]":
+ return pos + 1, array
+ if c != ",":
+ raise suffixed_err(src, pos, "Unclosed array")
+ pos += 1
+
+ pos = skip_comments_and_array_ws(src, pos)
+ if src.startswith("]", pos):
+ return pos + 1, array
+
+
+def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict]:
+ pos += 1
+ nested_dict = NestedDict()
+ flags = Flags()
+
+ pos = skip_chars(src, pos, TOML_WS)
+ if src.startswith("}", pos):
+ return pos + 1, nested_dict.dict
+ while True:
+ pos, key, value = parse_key_value_pair(src, pos, parse_float)
+ key_parent, key_stem = key[:-1], key[-1]
+ if flags.is_(key, Flags.FROZEN):
+ raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}")
+ try:
+ nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
+ except KeyError:
+ raise suffixed_err(src, pos, "Cannot overwrite a value") from None
+ if key_stem in nest:
+ raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}")
+ nest[key_stem] = value
+ pos = skip_chars(src, pos, TOML_WS)
+ c = src[pos : pos + 1]
+ if c == "}":
+ return pos + 1, nested_dict.dict
+ if c != ",":
+ raise suffixed_err(src, pos, "Unclosed inline table")
+ if isinstance(value, (dict, list)):
+ flags.set(key, Flags.FROZEN, recursive=True)
+ pos += 1
+ pos = skip_chars(src, pos, TOML_WS)
+
+
+def parse_basic_str_escape(
+ src: str, pos: Pos, *, multiline: bool = False
+) -> tuple[Pos, str]:
+ escape_id = src[pos : pos + 2]
+ pos += 2
+ if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}:
+ # Skip whitespace until next non-whitespace character or end of
+ # the doc. Error if non-whitespace is found before newline.
+ if escape_id != "\\\n":
+ pos = skip_chars(src, pos, TOML_WS)
+ try:
+ char = src[pos]
+ except IndexError:
+ return pos, ""
+ if char != "\n":
+ raise suffixed_err(src, pos, "Unescaped '\\' in a string")
+ pos += 1
+ pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
+ return pos, ""
+ if escape_id == "\\u":
+ return parse_hex_char(src, pos, 4)
+ if escape_id == "\\U":
+ return parse_hex_char(src, pos, 8)
+ try:
+ return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
+ except KeyError:
+ raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None
+
+
+def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]:
+ return parse_basic_str_escape(src, pos, multiline=True)
+
+
+def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]:
+ hex_str = src[pos : pos + hex_len]
+ if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str):
+ raise suffixed_err(src, pos, "Invalid hex value")
+ pos += hex_len
+ hex_int = int(hex_str, 16)
+ if not is_unicode_scalar_value(hex_int):
+ raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value")
+ return pos, chr(hex_int)
+
+
+def parse_literal_str(src: str, pos: Pos) -> tuple[Pos, str]:
+ pos += 1 # Skip starting apostrophe
+ start_pos = pos
+ pos = skip_until(
+ src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True
+ )
+ return pos + 1, src[start_pos:pos] # Skip ending apostrophe
+
+
+def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> tuple[Pos, str]:
+ pos += 3
+ if src.startswith("\n", pos):
+ pos += 1
+
+ if literal:
+ delim = "'"
+ end_pos = skip_until(
+ src,
+ pos,
+ "'''",
+ error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
+ error_on_eof=True,
+ )
+ result = src[pos:end_pos]
+ pos = end_pos + 3
+ else:
+ delim = '"'
+ pos, result = parse_basic_str(src, pos, multiline=True)
+
+ # Add at maximum two extra apostrophes/quotes if the end sequence
+ # is 4 or 5 chars long instead of just 3.
+ if not src.startswith(delim, pos):
+ return pos, result
+ pos += 1
+ if not src.startswith(delim, pos):
+ return pos, result + delim
+ pos += 1
+ return pos, result + (delim * 2)
+
+
+def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]:
+ if multiline:
+ error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS
+ parse_escapes = parse_basic_str_escape_multiline
+ else:
+ error_on = ILLEGAL_BASIC_STR_CHARS
+ parse_escapes = parse_basic_str_escape
+ result = ""
+ start_pos = pos
+ while True:
+ try:
+ char = src[pos]
+ except IndexError:
+ raise suffixed_err(src, pos, "Unterminated string") from None
+ if char == '"':
+ if not multiline:
+ return pos + 1, result + src[start_pos:pos]
+ if src.startswith('"""', pos):
+ return pos + 3, result + src[start_pos:pos]
+ pos += 1
+ continue
+ if char == "\\":
+ result += src[start_pos:pos]
+ pos, parsed_escape = parse_escapes(src, pos)
+ result += parsed_escape
+ start_pos = pos
+ continue
+ if char in error_on:
+ raise suffixed_err(src, pos, f"Illegal character {char!r}")
+ pos += 1
+
+
+def parse_value( # noqa: C901
+ src: str, pos: Pos, parse_float: ParseFloat
+) -> tuple[Pos, Any]:
+ try:
+ char: str | None = src[pos]
+ except IndexError:
+ char = None
+
+ # IMPORTANT: order conditions based on speed of checking and likelihood
+
+ # Basic strings
+ if char == '"':
+ if src.startswith('"""', pos):
+ return parse_multiline_str(src, pos, literal=False)
+ return parse_one_line_basic_str(src, pos)
+
+ # Literal strings
+ if char == "'":
+ if src.startswith("'''", pos):
+ return parse_multiline_str(src, pos, literal=True)
+ return parse_literal_str(src, pos)
+
+ # Booleans
+ if char == "t":
+ if src.startswith("true", pos):
+ return pos + 4, True
+ if char == "f":
+ if src.startswith("false", pos):
+ return pos + 5, False
+
+ # Arrays
+ if char == "[":
+ return parse_array(src, pos, parse_float)
+
+ # Inline tables
+ if char == "{":
+ return parse_inline_table(src, pos, parse_float)
+
+ # Dates and times
+ datetime_match = RE_DATETIME.match(src, pos)
+ if datetime_match:
+ try:
+ datetime_obj = match_to_datetime(datetime_match)
+ except ValueError as e:
+ raise suffixed_err(src, pos, "Invalid date or datetime") from e
+ return datetime_match.end(), datetime_obj
+ localtime_match = RE_LOCALTIME.match(src, pos)
+ if localtime_match:
+ return localtime_match.end(), match_to_localtime(localtime_match)
+
+ # Integers and "normal" floats.
+ # The regex will greedily match any type starting with a decimal
+ # char, so needs to be located after handling of dates and times.
+ number_match = RE_NUMBER.match(src, pos)
+ if number_match:
+ return number_match.end(), match_to_number(number_match, parse_float)
+
+ # Special floats
+ first_three = src[pos : pos + 3]
+ if first_three in {"inf", "nan"}:
+ return pos + 3, parse_float(first_three)
+ first_four = src[pos : pos + 4]
+ if first_four in {"-inf", "+inf", "-nan", "+nan"}:
+ return pos + 4, parse_float(first_four)
+
+ raise suffixed_err(src, pos, "Invalid value")
+
+
+def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError:
+ """Return a `TOMLDecodeError` where error message is suffixed with
+ coordinates in source."""
+
+ def coord_repr(src: str, pos: Pos) -> str:
+ if pos >= len(src):
+ return "end of document"
+ line = src.count("\n", 0, pos) + 1
+ if line == 1:
+ column = pos + 1
+ else:
+ column = pos - src.rindex("\n", 0, pos)
+ return f"line {line}, column {column}"
+
+ return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})")
+
+
+def is_unicode_scalar_value(codepoint: int) -> bool:
+ return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
+
+
+def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat:
+ """A decorator to make `parse_float` safe.
+
+ `parse_float` must not return dicts or lists, because these types
+ would be mixed with parsed TOML tables and arrays, thus confusing
+ the parser. The returned decorated callable raises `ValueError`
+ instead of returning illegal types.
+ """
+ # The default `float` callable never returns illegal types. Optimize it.
+ if parse_float is float: # type: ignore[comparison-overlap]
+ return float
+
+ def safe_parse_float(float_str: str) -> Any:
+ float_value = parse_float(float_str)
+ if isinstance(float_value, (dict, list)):
+ raise ValueError("parse_float must not return dicts or lists")
+ return float_value
+
+ return safe_parse_float
diff --git a/common/py3-stdlib/tomllib/_re.py b/common/py3-stdlib/tomllib/_re.py
new file mode 100644
index 0000000..994bb74
--- /dev/null
+++ b/common/py3-stdlib/tomllib/_re.py
@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+# Licensed to PSF under a Contributor Agreement.
+
+from __future__ import annotations
+
+from datetime import date, datetime, time, timedelta, timezone, tzinfo
+from functools import lru_cache
+import re
+from typing import Any
+
+from ._types import ParseFloat
+
+# E.g.
+# - 00:32:00.999999
+# - 00:32:00
+_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?"
+
+RE_NUMBER = re.compile(
+ r"""
+0
+(?:
+ x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
+ |
+ b[01](?:_?[01])* # bin
+ |
+ o[0-7](?:_?[0-7])* # oct
+)
+|
+[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
+(?P<floatpart>
+ (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
+ (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
+)
+""",
+ flags=re.VERBOSE,
+)
+RE_LOCALTIME = re.compile(_TIME_RE_STR)
+RE_DATETIME = re.compile(
+ rf"""
+([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
+(?:
+ [Tt ]
+ {_TIME_RE_STR}
+ (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
+)?
+""",
+ flags=re.VERBOSE,
+)
+
+
+def match_to_datetime(match: re.Match) -> datetime | date:
+ """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
+
+ Raises ValueError if the match does not correspond to a valid date
+ or datetime.
+ """
+ (
+ year_str,
+ month_str,
+ day_str,
+ hour_str,
+ minute_str,
+ sec_str,
+ micros_str,
+ zulu_time,
+ offset_sign_str,
+ offset_hour_str,
+ offset_minute_str,
+ ) = match.groups()
+ year, month, day = int(year_str), int(month_str), int(day_str)
+ if hour_str is None:
+ return date(year, month, day)
+ hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
+ micros = int(micros_str.ljust(6, "0")) if micros_str else 0
+ if offset_sign_str:
+ tz: tzinfo | None = cached_tz(
+ offset_hour_str, offset_minute_str, offset_sign_str
+ )
+ elif zulu_time:
+ tz = timezone.utc
+ else: # local date-time
+ tz = None
+ return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
+
+
+@lru_cache(maxsize=None)
+def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone:
+ sign = 1 if sign_str == "+" else -1
+ return timezone(
+ timedelta(
+ hours=sign * int(hour_str),
+ minutes=sign * int(minute_str),
+ )
+ )
+
+
+def match_to_localtime(match: re.Match) -> time:
+ hour_str, minute_str, sec_str, micros_str = match.groups()
+ micros = int(micros_str.ljust(6, "0")) if micros_str else 0
+ return time(int(hour_str), int(minute_str), int(sec_str), micros)
+
+
+def match_to_number(match: re.Match, parse_float: ParseFloat) -> Any:
+ if match.group("floatpart"):
+ return parse_float(match.group())
+ return int(match.group(), 0)
diff --git a/common/py3-stdlib/tomllib/_types.py b/common/py3-stdlib/tomllib/_types.py
new file mode 100644
index 0000000..d949412
--- /dev/null
+++ b/common/py3-stdlib/tomllib/_types.py
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+# Licensed to PSF under a Contributor Agreement.
+
+from typing import Any, Callable, Tuple
+
+# Type annotations
+ParseFloat = Callable[[str], Any]
+Key = Tuple[str, ...]
+Pos = int
diff --git a/common/py3-stdlib/trace.py b/common/py3-stdlib/trace.py
index 213e465..fb9a423 100755
--- a/common/py3-stdlib/trace.py
+++ b/common/py3-stdlib/trace.py
@@ -49,6 +49,7 @@
"""
__all__ = ['Trace', 'CoverageResults']
+import io
import linecache
import os
import sys
@@ -716,7 +717,7 @@
sys.argv = [opts.progname, *opts.arguments]
sys.path[0] = os.path.dirname(opts.progname)
- with open(opts.progname, 'rb') as fp:
+ with io.open_code(opts.progname) as fp:
code = compile(fp.read(), opts.progname, 'exec')
# try to emulate __main__ namespace as much as possible
globs = {
diff --git a/common/py3-stdlib/traceback.py b/common/py3-stdlib/traceback.py
index d6a010f..09950a8 100644
--- a/common/py3-stdlib/traceback.py
+++ b/common/py3-stdlib/traceback.py
@@ -1,9 +1,11 @@
"""Extract, format and print information about Python stack traces."""
-import collections
+import collections.abc
import itertools
import linecache
import sys
+import textwrap
+from contextlib import suppress
__all__ = ['extract_stack', 'extract_tb', 'format_exception',
'format_exception_only', 'format_list', 'format_stack',
@@ -69,7 +71,8 @@
trace. The line is a string with leading and trailing
whitespace stripped; if the source is not available it is None.
"""
- return StackSummary.extract(walk_tb(tb), limit=limit)
+ return StackSummary._extract_from_extended_frame_gen(
+ _walk_tb_with_full_positions(tb), limit=limit)
#
# Exception formatting and output.
@@ -95,7 +98,11 @@
raise ValueError("Both or neither of value and tb must be given")
if value is tb is _sentinel:
if exc is not None:
- return exc, exc.__traceback__
+ if isinstance(exc, BaseException):
+ return exc, exc.__traceback__
+
+ raise TypeError(f'Exception expected for value, '
+ f'{type(exc).__name__} found')
else:
return None, None
return value, tb
@@ -114,11 +121,8 @@
position of the error.
"""
value, tb = _parse_value_tb(exc, value, tb)
- if file is None:
- file = sys.stderr
te = TracebackException(type(value), value, tb, limit=limit, compact=True)
- for line in te.format(chain=chain):
- print(line, file=file, end="")
+ te.print(file=file, chain=chain)
def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \
@@ -159,18 +163,18 @@
# -- not official API but folk probably use these two functions.
def _format_final_exc_line(etype, value):
- valuestr = _some_str(value)
+ valuestr = _safe_string(value, 'exception')
if value is None or not valuestr:
line = "%s\n" % etype
else:
line = "%s: %s\n" % (etype, valuestr)
return line
-def _some_str(value):
+def _safe_string(value, what, func=str):
try:
- return str(value)
+ return func(value)
except:
- return '<unprintable %s object>' % type(value).__name__
+ return f'<{what} {func.__name__}() failed>'
# --
@@ -241,7 +245,7 @@
class FrameSummary:
- """A single frame from a traceback.
+ """Information about a single frame from a traceback.
- :attr:`filename` The filename for the frame.
- :attr:`lineno` The line within filename for the frame that was
@@ -254,10 +258,12 @@
mapping the name to the repr() of the variable.
"""
- __slots__ = ('filename', 'lineno', 'name', '_line', 'locals')
+ __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno',
+ 'name', '_line', 'locals')
def __init__(self, filename, lineno, name, *, lookup_line=True,
- locals=None, line=None):
+ locals=None, line=None,
+ end_lineno=None, colno=None, end_colno=None):
"""Construct a FrameSummary.
:param lookup_line: If True, `linecache` is consulted for the source
@@ -274,6 +280,9 @@
if lookup_line:
self.line
self.locals = {k: repr(v) for k, v in locals.items()} if locals else None
+ self.end_lineno = end_lineno
+ self.colno = colno
+ self.end_colno = end_colno
def __eq__(self, other):
if isinstance(other, FrameSummary):
@@ -299,6 +308,12 @@
return 4
@property
+ def _original_line(self):
+ # Returns the line as-is from the source, without modifying whitespace.
+ self.line
+ return self._line
+
+ @property
def line(self):
if self._line is None:
if self.lineno is None:
@@ -306,6 +321,7 @@
self._line = linecache.getline(self.filename, self.lineno)
return self._line.strip()
+
def walk_stack(f):
"""Walk a stack yielding the frame and line number for each frame.
@@ -313,7 +329,7 @@
current stack is used. Usually used with StackSummary.extract.
"""
if f is None:
- f = sys._getframe().f_back.f_back
+ f = sys._getframe().f_back.f_back.f_back.f_back
while f is not None:
yield f, f.f_lineno
f = f.f_back
@@ -330,18 +346,39 @@
tb = tb.tb_next
+def _walk_tb_with_full_positions(tb):
+ # Internal version of walk_tb that yields full code positions including
+ # end line and column information.
+ while tb is not None:
+ positions = _get_code_position(tb.tb_frame.f_code, tb.tb_lasti)
+ # Yield tb_lineno when co_positions does not have a line number to
+ # maintain behavior with walk_tb.
+ if positions[0] is None:
+ yield tb.tb_frame, (tb.tb_lineno, ) + positions[1:]
+ else:
+ yield tb.tb_frame, positions
+ tb = tb.tb_next
+
+
+def _get_code_position(code, instruction_index):
+ if instruction_index < 0:
+ return (None, None, None, None)
+ positions_gen = code.co_positions()
+ return next(itertools.islice(positions_gen, instruction_index // 2, None))
+
+
_RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c.
class StackSummary(list):
- """A stack of frames."""
+ """A list of FrameSummary objects, representing a stack of frames."""
@classmethod
def extract(klass, frame_gen, *, limit=None, lookup_lines=True,
capture_locals=False):
"""Create a StackSummary from a traceback or stack object.
- :param frame_gen: A generator that yields (frame, lineno) tuples to
- include in the stack.
+ :param frame_gen: A generator that yields (frame, lineno) tuples
+ whose summaries are to be included in the stack.
:param limit: None to include all frames or the number of frames to
include.
:param lookup_lines: If True, lookup lines for each frame immediately,
@@ -349,6 +386,21 @@
:param capture_locals: If True, the local variables from each frame will
be captured as object representations into the FrameSummary.
"""
+ def extended_frame_gen():
+ for f, lineno in frame_gen:
+ yield f, (lineno, None, None, None)
+
+ return klass._extract_from_extended_frame_gen(
+ extended_frame_gen(), limit=limit, lookup_lines=lookup_lines,
+ capture_locals=capture_locals)
+
+ @classmethod
+ def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None,
+ lookup_lines=True, capture_locals=False):
+ # Same as extract but operates on a frame generator that yields
+ # (frame, (lineno, end_lineno, colno, end_colno)) in the stack.
+ # Only lineno is required, the remaining fields can be None if the
+ # information is not available.
if limit is None:
limit = getattr(sys, 'tracebacklimit', None)
if limit is not None and limit < 0:
@@ -361,7 +413,7 @@
result = klass()
fnames = set()
- for f, lineno in frame_gen:
+ for f, (lineno, end_lineno, colno, end_colno) in frame_gen:
co = f.f_code
filename = co.co_filename
name = co.co_name
@@ -374,7 +426,8 @@
else:
f_locals = None
result.append(FrameSummary(
- filename, lineno, name, lookup_line=False, locals=f_locals))
+ filename, lineno, name, lookup_line=False, locals=f_locals,
+ end_lineno=end_lineno, colno=colno, end_colno=end_colno))
for filename in fnames:
linecache.checkcache(filename)
# If immediate lookup was desired, trigger lookups now.
@@ -402,6 +455,61 @@
result.append(FrameSummary(filename, lineno, name, line=line))
return result
+ def format_frame_summary(self, frame_summary):
+ """Format the lines for a single FrameSummary.
+
+ Returns a string representing one frame involved in the stack. This
+ gets called for every frame to be printed in the stack summary.
+ """
+ row = []
+ row.append(' File "{}", line {}, in {}\n'.format(
+ frame_summary.filename, frame_summary.lineno, frame_summary.name))
+ if frame_summary.line:
+ stripped_line = frame_summary.line.strip()
+ row.append(' {}\n'.format(stripped_line))
+
+ orig_line_len = len(frame_summary._original_line)
+ frame_line_len = len(frame_summary.line.lstrip())
+ stripped_characters = orig_line_len - frame_line_len
+ if (
+ frame_summary.colno is not None
+ and frame_summary.end_colno is not None
+ ):
+ start_offset = _byte_offset_to_character_offset(
+ frame_summary._original_line, frame_summary.colno) + 1
+ end_offset = _byte_offset_to_character_offset(
+ frame_summary._original_line, frame_summary.end_colno) + 1
+
+ anchors = None
+ if frame_summary.lineno == frame_summary.end_lineno:
+ with suppress(Exception):
+ anchors = _extract_caret_anchors_from_line_segment(
+ frame_summary._original_line[start_offset - 1:end_offset - 1]
+ )
+ else:
+ end_offset = stripped_characters + len(stripped_line)
+
+ # show indicators if primary char doesn't span the frame line
+ if end_offset - start_offset < len(stripped_line) or (
+ anchors and anchors.right_start_offset - anchors.left_end_offset > 0):
+ row.append(' ')
+ row.append(' ' * (start_offset - stripped_characters))
+
+ if anchors:
+ row.append(anchors.primary_char * (anchors.left_end_offset))
+ row.append(anchors.secondary_char * (anchors.right_start_offset - anchors.left_end_offset))
+ row.append(anchors.primary_char * (end_offset - start_offset - anchors.right_start_offset))
+ else:
+ row.append('^' * (end_offset - start_offset))
+
+ row.append('\n')
+
+ if frame_summary.locals:
+ for name, value in sorted(frame_summary.locals.items()):
+ row.append(' {name} = {value}\n'.format(name=name, value=value))
+
+ return ''.join(row)
+
def format(self):
"""Format the stack ready for printing.
@@ -419,32 +527,28 @@
last_line = None
last_name = None
count = 0
- for frame in self:
- if (last_file is None or last_file != frame.filename or
- last_line is None or last_line != frame.lineno or
- last_name is None or last_name != frame.name):
+ for frame_summary in self:
+ formatted_frame = self.format_frame_summary(frame_summary)
+ if formatted_frame is None:
+ continue
+ if (last_file is None or last_file != frame_summary.filename or
+ last_line is None or last_line != frame_summary.lineno or
+ last_name is None or last_name != frame_summary.name):
if count > _RECURSIVE_CUTOFF:
count -= _RECURSIVE_CUTOFF
result.append(
f' [Previous line repeated {count} more '
f'time{"s" if count > 1 else ""}]\n'
)
- last_file = frame.filename
- last_line = frame.lineno
- last_name = frame.name
+ last_file = frame_summary.filename
+ last_line = frame_summary.lineno
+ last_name = frame_summary.name
count = 0
count += 1
if count > _RECURSIVE_CUTOFF:
continue
- row = []
- row.append(' File "{}", line {}, in {}\n'.format(
- frame.filename, frame.lineno, frame.name))
- if frame.line:
- row.append(' {}\n'.format(frame.line.strip()))
- if frame.locals:
- for name, value in sorted(frame.locals.items()):
- row.append(' {name} = {value}\n'.format(name=name, value=value))
- result.append(''.join(row))
+ result.append(formatted_frame)
+
if count > _RECURSIVE_CUTOFF:
count -= _RECURSIVE_CUTOFF
result.append(
@@ -454,6 +558,83 @@
return result
+def _byte_offset_to_character_offset(str, offset):
+ as_utf8 = str.encode('utf-8')
+ return len(as_utf8[:offset].decode("utf-8", errors="replace"))
+
+
+_Anchors = collections.namedtuple(
+ "_Anchors",
+ [
+ "left_end_offset",
+ "right_start_offset",
+ "primary_char",
+ "secondary_char",
+ ],
+ defaults=["~", "^"]
+)
+
+def _extract_caret_anchors_from_line_segment(segment):
+ import ast
+
+ try:
+ tree = ast.parse(segment)
+ except SyntaxError:
+ return None
+
+ if len(tree.body) != 1:
+ return None
+
+ normalize = lambda offset: _byte_offset_to_character_offset(segment, offset)
+ statement = tree.body[0]
+ match statement:
+ case ast.Expr(expr):
+ match expr:
+ case ast.BinOp():
+ operator_start = normalize(expr.left.end_col_offset)
+ operator_end = normalize(expr.right.col_offset)
+ operator_str = segment[operator_start:operator_end]
+ operator_offset = len(operator_str) - len(operator_str.lstrip())
+
+ left_anchor = expr.left.end_col_offset + operator_offset
+ right_anchor = left_anchor + 1
+ if (
+ operator_offset + 1 < len(operator_str)
+ and not operator_str[operator_offset + 1].isspace()
+ ):
+ right_anchor += 1
+ return _Anchors(normalize(left_anchor), normalize(right_anchor))
+ case ast.Subscript():
+ subscript_start = normalize(expr.value.end_col_offset)
+ subscript_end = normalize(expr.slice.end_col_offset + 1)
+ return _Anchors(subscript_start, subscript_end)
+
+ return None
+
+
+class _ExceptionPrintContext:
+ def __init__(self):
+ self.seen = set()
+ self.exception_group_depth = 0
+ self.need_close = False
+
+ def indent(self):
+ return ' ' * (2 * self.exception_group_depth)
+
+ def emit(self, text_gen, margin_char=None):
+ if margin_char is None:
+ margin_char = '|'
+ indent_str = self.indent()
+ if self.exception_group_depth:
+ indent_str += margin_char + ' '
+
+ if isinstance(text_gen, str):
+ yield textwrap.indent(text_gen, indent_str, lambda line: True)
+ else:
+ for text in text_gen:
+ yield textwrap.indent(text, indent_str, lambda line: True)
+
+
class TracebackException:
"""An exception ready for rendering.
@@ -461,12 +642,19 @@
to this intermediary form to ensure that no references are held, while
still being able to fully print or format it.
+ max_group_width and max_group_depth control the formatting of exception
+ groups. The depth refers to the nesting level of the group, and the width
+ refers to the size of a single exception group's exceptions array. The
+ formatted output is truncated when either limit is exceeded.
+
Use `from_exception` to create TracebackException instances from exception
objects, or the constructor to create TracebackException instances from
individual components.
- :attr:`__cause__` A TracebackException of the original *__cause__*.
- :attr:`__context__` A TracebackException of the original *__context__*.
+ - :attr:`exceptions` For exception groups - a list of TracebackException
+ instances for the nested *exceptions*. ``None`` for other exceptions.
- :attr:`__suppress_context__` The *__suppress_context__* value from the
original exception.
- :attr:`stack` A `StackSummary` representing the traceback.
@@ -481,14 +669,14 @@
occurred.
- :attr:`offset` For syntax errors - the offset into the text where the
error occurred.
- - :attr:`end_offset` For syntax errors - the offset into the text where the
- error occurred. Can be `None` if not present.
+ - :attr:`end_offset` For syntax errors - the end offset into the text where
+ the error occurred. Can be `None` if not present.
- :attr:`msg` For syntax errors - the compiler error message.
"""
def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None,
lookup_lines=True, capture_locals=False, compact=False,
- _seen=None):
+ max_group_width=15, max_group_depth=10, _seen=None):
# NB: we need to accept exc_traceback, exc_value, exc_traceback to
# permit backwards compat with the existing API, otherwise we
# need stub thunk objects just to glue it together.
@@ -498,14 +686,19 @@
_seen = set()
_seen.add(id(exc_value))
- # TODO: locals.
- self.stack = StackSummary.extract(
- walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines,
+ self.max_group_width = max_group_width
+ self.max_group_depth = max_group_depth
+
+ self.stack = StackSummary._extract_from_extended_frame_gen(
+ _walk_tb_with_full_positions(exc_traceback),
+ limit=limit, lookup_lines=lookup_lines,
capture_locals=capture_locals)
self.exc_type = exc_type
# Capture now to permit freeing resources: only complication is in the
# unofficial API _format_final_exc_line
- self._str = _some_str(exc_value)
+ self._str = _safe_string(exc_value, 'exception')
+ self.__notes__ = getattr(exc_value, '__notes__', None)
+
if exc_type and issubclass(exc_type, SyntaxError):
# Handle SyntaxError's specially
self.filename = exc_value.filename
@@ -537,6 +730,8 @@
limit=limit,
lookup_lines=lookup_lines,
capture_locals=capture_locals,
+ max_group_width=max_group_width,
+ max_group_depth=max_group_depth,
_seen=_seen)
else:
cause = None
@@ -556,15 +751,38 @@
limit=limit,
lookup_lines=lookup_lines,
capture_locals=capture_locals,
+ max_group_width=max_group_width,
+ max_group_depth=max_group_depth,
_seen=_seen)
else:
context = None
+
+ if e and isinstance(e, BaseExceptionGroup):
+ exceptions = []
+ for exc in e.exceptions:
+ texc = TracebackException(
+ type(exc),
+ exc,
+ exc.__traceback__,
+ limit=limit,
+ lookup_lines=lookup_lines,
+ capture_locals=capture_locals,
+ max_group_width=max_group_width,
+ max_group_depth=max_group_depth,
+ _seen=_seen)
+ exceptions.append(texc)
+ else:
+ exceptions = None
+
te.__cause__ = cause
te.__context__ = context
+ te.exceptions = exceptions
if cause:
queue.append((te.__cause__, e.__cause__))
if context:
queue.append((te.__context__, e.__context__))
+ if exceptions:
+ queue.extend(zip(te.exceptions, e.exceptions))
@classmethod
def from_exception(cls, exc, *args, **kwargs):
@@ -612,6 +830,12 @@
yield _format_final_exc_line(stype, self._str)
else:
yield from self._format_syntax_error(stype)
+ if isinstance(self.__notes__, collections.abc.Sequence):
+ for note in self.__notes__:
+ note = _safe_string(note, 'note')
+ yield from [l + '\n' for l in note.split('\n')]
+ elif self.__notes__ is not None:
+ yield _safe_string(self.__notes__, '__notes__', func=repr)
def _format_syntax_error(self, stype):
"""Format SyntaxError exceptions (internal helper)."""
@@ -649,7 +873,7 @@
msg = self.msg or "<no detail available>"
yield "{}: {}{}\n".format(stype, msg, filename_suffix)
- def format(self, *, chain=True):
+ def format(self, *, chain=True, _ctx=None):
"""Format the exception.
If chain is not *True*, *__cause__* and *__context__* will not be formatted.
@@ -662,10 +886,13 @@
string in the output.
"""
+ if _ctx is None:
+ _ctx = _ExceptionPrintContext()
+
output = []
exc = self
- while exc:
- if chain:
+ if chain:
+ while exc:
if exc.__cause__ is not None:
chained_msg = _cause_message
chained_exc = exc.__cause__
@@ -679,14 +906,77 @@
output.append((chained_msg, exc))
exc = chained_exc
- else:
- output.append((None, exc))
- exc = None
+ else:
+ output.append((None, exc))
for msg, exc in reversed(output):
if msg is not None:
- yield msg
- if exc.stack:
- yield 'Traceback (most recent call last):\n'
- yield from exc.stack.format()
- yield from exc.format_exception_only()
+ yield from _ctx.emit(msg)
+ if exc.exceptions is None:
+ if exc.stack:
+ yield from _ctx.emit('Traceback (most recent call last):\n')
+ yield from _ctx.emit(exc.stack.format())
+ yield from _ctx.emit(exc.format_exception_only())
+ elif _ctx.exception_group_depth > self.max_group_depth:
+ # exception group, but depth exceeds limit
+ yield from _ctx.emit(
+ f"... (max_group_depth is {self.max_group_depth})\n")
+ else:
+ # format exception group
+ is_toplevel = (_ctx.exception_group_depth == 0)
+ if is_toplevel:
+ _ctx.exception_group_depth += 1
+
+ if exc.stack:
+ yield from _ctx.emit(
+ 'Exception Group Traceback (most recent call last):\n',
+ margin_char = '+' if is_toplevel else None)
+ yield from _ctx.emit(exc.stack.format())
+
+ yield from _ctx.emit(exc.format_exception_only())
+ num_excs = len(exc.exceptions)
+ if num_excs <= self.max_group_width:
+ n = num_excs
+ else:
+ n = self.max_group_width + 1
+ _ctx.need_close = False
+ for i in range(n):
+ last_exc = (i == n-1)
+ if last_exc:
+ # The closing frame may be added by a recursive call
+ _ctx.need_close = True
+
+ if self.max_group_width is not None:
+ truncated = (i >= self.max_group_width)
+ else:
+ truncated = False
+ title = f'{i+1}' if not truncated else '...'
+ yield (_ctx.indent() +
+ ('+-' if i==0 else ' ') +
+ f'+---------------- {title} ----------------\n')
+ _ctx.exception_group_depth += 1
+ if not truncated:
+ yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx)
+ else:
+ remaining = num_excs - self.max_group_width
+ plural = 's' if remaining > 1 else ''
+ yield from _ctx.emit(
+ f"and {remaining} more exception{plural}\n")
+
+ if last_exc and _ctx.need_close:
+ yield (_ctx.indent() +
+ "+------------------------------------\n")
+ _ctx.need_close = False
+ _ctx.exception_group_depth -= 1
+
+ if is_toplevel:
+ assert _ctx.exception_group_depth == 1
+ _ctx.exception_group_depth = 0
+
+
+ def print(self, *, file=None, chain=True):
+ """Print the result of self.format(chain=chain) to 'file'."""
+ if file is None:
+ file = sys.stderr
+ for line in self.format(chain=chain):
+ print(line, file=file, end="")
diff --git a/common/py3-stdlib/turtle.py b/common/py3-stdlib/turtle.py
index d287c15..1b36932 100644
--- a/common/py3-stdlib/turtle.py
+++ b/common/py3-stdlib/turtle.py
@@ -110,6 +110,7 @@
import time
import inspect
import sys
+import warnings
from os.path import isfile, split, join
from copy import deepcopy
@@ -2849,20 +2850,23 @@
regardless of its current tilt-angle. DO NOT change the turtle's
heading (direction of movement).
+ Deprecated since Python 3.1
Examples (for a Turtle instance named turtle):
>>> turtle.shape("circle")
>>> turtle.shapesize(5,2)
>>> turtle.settiltangle(45)
- >>> stamp()
+ >>> turtle.stamp()
>>> turtle.fd(50)
>>> turtle.settiltangle(-45)
- >>> stamp()
+ >>> turtle.stamp()
>>> turtle.fd(50)
"""
- tilt = -angle * self._degreesPerAU * self._angleOrient
- tilt = math.radians(tilt) % math.tau
- self.pen(resizemode="user", tilt=tilt)
+ warnings._deprecated("turtle.RawTurtle.settiltangle()",
+ "{name!r} is deprecated since Python 3.1 and scheduled "
+ "for removal in Python {remove}. Use tiltangle() instead.",
+ remove=(3, 13))
+ self.tiltangle(angle)
def tiltangle(self, angle=None):
"""Set or return the current tilt-angle.
@@ -2881,15 +2885,27 @@
Examples (for a Turtle instance named turtle):
>>> turtle.shape("circle")
- >>> turtle.shapesize(5,2)
- >>> turtle.tilt(45)
+ >>> turtle.shapesize(5, 2)
>>> turtle.tiltangle()
+ 0.0
+ >>> turtle.tiltangle(45)
+ >>> turtle.tiltangle()
+ 45.0
+ >>> turtle.stamp()
+ >>> turtle.fd(50)
+ >>> turtle.tiltangle(-45)
+ >>> turtle.tiltangle()
+ 315.0
+ >>> turtle.stamp()
+ >>> turtle.fd(50)
"""
if angle is None:
tilt = -math.degrees(self._tilt) * self._angleOrient
return (tilt / self._degreesPerAU) % self._fullcircle
else:
- self.settiltangle(angle)
+ tilt = -angle * self._degreesPerAU * self._angleOrient
+ tilt = math.radians(tilt) % math.tau
+ self.pen(resizemode="user", tilt=tilt)
def tilt(self, angle):
"""Rotate the turtleshape by angle.
@@ -2908,7 +2924,7 @@
>>> turtle.tilt(30)
>>> turtle.fd(50)
"""
- self.settiltangle(angle + self.tiltangle())
+ self.tiltangle(angle + self.tiltangle())
def shapetransform(self, t11=None, t12=None, t21=None, t22=None):
"""Set or return the current transformation matrix of the turtle shape.
@@ -3827,7 +3843,7 @@
default value is turtle_docstringdict
Has to be called explicitly, (not used by the turtle-graphics classes)
- The docstring dictionary will be written to the Python script <filname>.py
+ The docstring dictionary will be written to the Python script <filename>.py
It is intended to serve as a template for translation of the docstrings
into different languages.
"""
diff --git a/common/py3-stdlib/types.py b/common/py3-stdlib/types.py
index 62122a9..2e73fbc 100644
--- a/common/py3-stdlib/types.py
+++ b/common/py3-stdlib/types.py
@@ -52,11 +52,9 @@
try:
raise TypeError
-except TypeError:
- tb = sys.exc_info()[2]
- TracebackType = type(tb)
- FrameType = type(tb.tb_frame)
- tb = None; del tb
+except TypeError as exc:
+ TracebackType = type(exc.__traceback__)
+ FrameType = type(exc.__traceback__.tb_frame)
# For Jython, the following two types are identical
GetSetDescriptorType = type(FunctionType.__code__)
@@ -82,7 +80,7 @@
updated = False
shift = 0
for i, base in enumerate(bases):
- if isinstance(base, type) and not isinstance(base, GenericAlias):
+ if isinstance(base, type):
continue
if not hasattr(base, "__mro_entries__"):
continue
@@ -158,7 +156,7 @@
attributes on the class with the same name. (Enum used this between Python
versions 3.4 - 3.9 .)
- Subclass from this to use a different method of accessing virtual atributes
+ Subclass from this to use a different method of accessing virtual attributes
and still be treated properly by the inspect module. (Enum uses this since
Python 3.10 .)
diff --git a/common/py3-stdlib/typing.py b/common/py3-stdlib/typing.py
index 03cf243..e6ef344 100644
--- a/common/py3-stdlib/typing.py
+++ b/common/py3-stdlib/typing.py
@@ -1,26 +1,27 @@
"""
-The typing module: Support for gradual typing as defined by PEP 484.
+The typing module: Support for gradual typing as defined by PEP 484 and subsequent PEPs.
-At large scale, the structure of the module is following:
-* Imports and exports, all public names should be explicitly added to __all__.
-* Internal helper functions: these should never be used in code outside this module.
-* _SpecialForm and its instances (special forms):
- Any, NoReturn, ClassVar, Union, Optional, Concatenate
-* Classes whose instances can be type arguments in addition to types:
- ForwardRef, TypeVar and ParamSpec
-* The core of internal generics API: _GenericAlias and _VariadicGenericAlias, the latter is
- currently only used by Tuple and Callable. All subscripted types like X[int], Union[int, str],
- etc., are instances of either of these classes.
-* The public counterpart of the generics API consists of two classes: Generic and Protocol.
-* Public helper functions: get_type_hints, overload, cast, no_type_check,
- no_type_check_decorator.
-* Generic aliases for collections.abc ABCs and few additional protocols.
+Any name not present in __all__ is an implementation detail
+that may be changed without notice. Use at your own risk!
+
+Among other things, the module includes the following:
+* Generic, Protocol, and internal machinery to support generic aliases.
+ All subscripted types like X[int], Union[int, str] are generic aliases.
+* Various "special forms" that have unique meanings in type annotations:
+ NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others.
+* Classes whose instances can be type arguments to generic classes and functions:
+ TypeVar, ParamSpec, TypeVarTuple.
+* Public helper functions: get_type_hints, overload, cast, final, and others.
+* Several protocols to support duck-typing:
+ SupportsFloat, SupportsIndex, SupportsAbs, and others.
* Special types: NewType, NamedTuple, TypedDict.
-* Wrapper submodules for re and io related types.
+* Deprecated wrapper submodules for re and io related types.
+* Deprecated aliases for builtin types and collections.abc ABCs.
"""
from abc import abstractmethod, ABCMeta
import collections
+from collections import defaultdict
import collections.abc
import contextlib
import functools
@@ -28,8 +29,16 @@
import re as stdlib_re # Avoid confusion with the re we export.
import sys
import types
+import warnings
from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias
+
+try:
+ from _typing import _idfunc
+except ImportError:
+ def _idfunc(_, x):
+ return x
+
# Please keep __all__ alphabetized within each category.
__all__ = [
# Super-special typing primitives.
@@ -48,6 +57,7 @@
'Tuple',
'Type',
'TypeVar',
+ 'TypeVarTuple',
'Union',
# ABCs (from collections.abc).
@@ -109,24 +119,36 @@
# One-off things.
'AnyStr',
+ 'assert_type',
+ 'assert_never',
'cast',
+ 'clear_overloads',
+ 'dataclass_transform',
'final',
'get_args',
'get_origin',
+ 'get_overloads',
'get_type_hints',
'is_typeddict',
+ 'LiteralString',
+ 'Never',
'NewType',
'no_type_check',
'no_type_check_decorator',
'NoReturn',
+ 'NotRequired',
'overload',
'ParamSpecArgs',
'ParamSpecKwargs',
+ 'Required',
+ 'reveal_type',
'runtime_checkable',
+ 'Self',
'Text',
'TYPE_CHECKING',
'TypeAlias',
'TypeGuard',
+ 'Unpack',
]
# The pseudo-submodules 're' and 'io' are part of the public
@@ -149,7 +171,7 @@
As a special case, accept None and return type(None) instead. Also wrap strings
into ForwardRef instances. Consider several corner cases, for example plain
special forms like Union are not valid, while Union[int, str] is OK, etc.
- The msg argument is a human-readable error message, e.g::
+ The msg argument is a human-readable error message, e.g.::
"Union[arg, ...]: arg should be a type."
@@ -165,14 +187,13 @@
if (isinstance(arg, _GenericAlias) and
arg.__origin__ in invalid_generic_forms):
raise TypeError(f"{arg} is not valid as type argument")
- if arg in (Any, NoReturn, Final, TypeAlias):
+ if arg in (Any, LiteralString, NoReturn, Never, Self, TypeAlias):
+ return arg
+ if allow_special_forms and arg in (ClassVar, Final):
return arg
if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol):
raise TypeError(f"Plain {arg} is not valid as type argument")
- if isinstance(arg, (type, TypeVar, ForwardRef, types.UnionType, ParamSpec,
- ParamSpecArgs, ParamSpecKwargs)):
- return arg
- if not callable(arg):
+ if type(arg) is tuple:
raise TypeError(f"{msg} Got {arg!r:.100}.")
return arg
@@ -182,6 +203,24 @@
(tuple, list, ParamSpec, _ConcatenateGenericAlias))
+def _should_unflatten_callable_args(typ, args):
+ """Internal helper for munging collections.abc.Callable's __args__.
+
+ The canonical representation for a Callable's __args__ flattens the
+ argument types, see https://bugs.python.org/issue42195. For example::
+
+ collections.abc.Callable[[int, int], str].__args__ == (int, int, str)
+ collections.abc.Callable[ParamSpec, str].__args__ == (ParamSpec, str)
+
+ As a result, if we need to reconstruct the Callable from its __args__,
+ we need to unflatten it.
+ """
+ return (
+ typ.__origin__ is collections.abc.Callable
+ and not (len(args) == 2 and _is_param_expr(args[0]))
+ )
+
+
def _type_repr(obj):
"""Return the repr() of an object, special-casing types (internal helper).
@@ -203,25 +242,39 @@
return repr(obj)
-def _collect_type_vars(types_, typevar_types=None):
- """Collect all type variable contained
- in types in order of first appearance (lexicographic order). For example::
+def _collect_parameters(args):
+ """Collect all type variables and parameter specifications in args
+ in order of first appearance (lexicographic order).
- _collect_type_vars((T, List[S, T])) == (T, S)
+ For example::
+
+ assert _collect_parameters((T, Callable[P, T])) == (T, P)
"""
- if typevar_types is None:
- typevar_types = TypeVar
- tvars = []
- for t in types_:
- if isinstance(t, typevar_types) and t not in tvars:
- tvars.append(t)
- if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)):
- tvars.extend([t for t in t.__parameters__ if t not in tvars])
- return tuple(tvars)
+ parameters = []
+ for t in args:
+ if isinstance(t, type):
+ # We don't want __parameters__ descriptor of a bare Python class.
+ pass
+ elif isinstance(t, tuple):
+ # `t` might be a tuple, when `ParamSpec` is substituted with
+ # `[T, int]`, or `[int, *Ts]`, etc.
+ for x in t:
+ for collected in _collect_parameters([x]):
+ if collected not in parameters:
+ parameters.append(collected)
+ elif hasattr(t, '__typing_subst__'):
+ if t not in parameters:
+ parameters.append(t)
+ else:
+ for x in getattr(t, '__parameters__', ()):
+ if x not in parameters:
+ parameters.append(x)
+ return tuple(parameters)
def _check_generic(cls, parameters, elen):
"""Check correct count for parameters of a generic cls (internal helper).
+
This gives a nice error message in case of count mismatch.
"""
if not elen:
@@ -231,24 +284,15 @@
raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};"
f" actual {alen}, expected {elen}")
-def _prepare_paramspec_params(cls, params):
- """Prepares the parameters for a Generic containing ParamSpec
- variables (internal helper).
- """
- # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612.
- if (len(cls.__parameters__) == 1
- and params and not _is_param_expr(params[0])):
- assert isinstance(cls.__parameters__[0], ParamSpec)
- return (params,)
- else:
- _check_generic(cls, params, len(cls.__parameters__))
- _params = []
- # Convert lists to tuples to help other libraries cache the results.
- for p, tvar in zip(params, cls.__parameters__):
- if isinstance(tvar, ParamSpec) and isinstance(p, list):
- p = tuple(p)
- _params.append(p)
- return tuple(_params)
+def _unpack_args(args):
+ newargs = []
+ for arg in args:
+ subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
+ if subargs is not None and not (subargs and subargs[-1] is ...):
+ newargs.extend(subargs)
+ else:
+ newargs.append(arg)
+ return newargs
def _deduplicate(params):
# Weed out strict duplicates, preserving the first of each occurrence.
@@ -265,16 +309,15 @@
def _remove_dups_flatten(parameters):
- """An internal helper for Union creation and substitution: flatten Unions
- among parameters, then remove duplicates.
+ """Internal helper for Union creation and substitution.
+
+ Flatten Unions among parameters, then remove duplicates.
"""
# Flatten out Union[Union[...], ...].
params = []
for p in parameters:
if isinstance(p, (_UnionGenericAlias, types.UnionType)):
params.extend(p.__args__)
- elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union:
- params.extend(p[1:])
else:
params.append(p)
@@ -282,7 +325,7 @@
def _flatten_literal_params(parameters):
- """An internal helper for Literal creation: flatten Literals among parameters"""
+ """Internal helper for Literal creation: flatten Literals among parameters."""
params = []
for p in parameters:
if isinstance(p, _LiteralGenericAlias):
@@ -319,6 +362,7 @@
def _eval_type(t, globalns, localns, recursive_guard=frozenset()):
"""Evaluate all forward references in the given type t.
+
For use of globalns and localns see the docstring for get_type_hints().
recursive_guard is used to prevent infinite recursion with a recursive
ForwardRef.
@@ -326,6 +370,18 @@
if isinstance(t, ForwardRef):
return t._evaluate(globalns, localns, recursive_guard)
if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)):
+ if isinstance(t, GenericAlias):
+ args = tuple(
+ ForwardRef(arg) if isinstance(arg, str) else arg
+ for arg in t.__args__
+ )
+ is_unpacked = t.__unpacked__
+ if _should_unflatten_callable_args(t, args):
+ t = t.__origin__[(args[:-1], args[-1])]
+ else:
+ t = t.__origin__[args]
+ if is_unpacked:
+ t = Unpack[t]
ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__)
if ev_args == t.__args__:
return t
@@ -339,16 +395,17 @@
class _Final:
- """Mixin to prohibit subclassing"""
+ """Mixin to prohibit subclassing."""
__slots__ = ('__weakref__',)
- def __init_subclass__(self, /, *args, **kwds):
+ def __init_subclass__(cls, /, *args, **kwds):
if '_root' not in kwds:
raise TypeError("Cannot subclass special typing classes")
class _Immutable:
"""Mixin to indicate that object should not be copied."""
+
__slots__ = ()
def __copy__(self):
@@ -358,9 +415,27 @@
return self
+class _NotIterable:
+ """Mixin to prevent iteration, without being compatible with Iterable.
+
+ That is, we could do::
+
+ def __iter__(self): raise TypeError()
+
+ But this would make users of this mixin duck type-compatible with
+ collections.abc.Iterable - isinstance(foo, Iterable) would be True.
+
+ Luckily, we can instead prevent iteration by setting __iter__ to None, which
+ is treated specially.
+ """
+
+ __slots__ = ()
+ __iter__ = None
+
+
# Internal indicator of special typing constructs.
# See __doc__ instance attribute for specific docs.
-class _SpecialForm(_Final, _root=True):
+class _SpecialForm(_Final, _NotIterable, _root=True):
__slots__ = ('_name', '__doc__', '_getitem')
def __init__(self, getitem):
@@ -410,8 +485,19 @@
return self._getitem(self, *parameters)
-@_SpecialForm
-def Any(self, parameters):
+class _AnyMeta(type):
+ def __instancecheck__(self, obj):
+ if self is Any:
+ raise TypeError("typing.Any cannot be used with isinstance()")
+ return super().__instancecheck__(obj)
+
+ def __repr__(self):
+ if self is Any:
+ return "typing.Any"
+ return super().__repr__() # respect to subclasses
+
+
+class Any(metaclass=_AnyMeta):
"""Special type indicating an unconstrained type.
- Any is compatible with every type.
@@ -420,25 +506,108 @@
Note that all the above statements are true from the point of view of
static type checkers. At runtime, Any should not be used with instance
- or class checks.
+ checks.
"""
- raise TypeError(f"{self} is not subscriptable")
+
+ def __new__(cls, *args, **kwargs):
+ if cls is Any:
+ raise TypeError("Any cannot be instantiated")
+ return super().__new__(cls, *args, **kwargs)
+
@_SpecialForm
def NoReturn(self, parameters):
"""Special type indicating functions that never return.
+
Example::
- from typing import NoReturn
+ from typing import NoReturn
- def stop() -> NoReturn:
- raise Exception('no way')
+ def stop() -> NoReturn:
+ raise Exception('no way')
- This type is invalid in other positions, e.g., ``List[NoReturn]``
- will fail in static type checkers.
+ NoReturn can also be used as a bottom type, a type that
+ has no values. Starting in Python 3.11, the Never type should
+ be used for this concept instead. Type checkers should treat the two
+ equivalently.
"""
raise TypeError(f"{self} is not subscriptable")
+# This is semantically identical to NoReturn, but it is implemented
+# separately so that type checkers can distinguish between the two
+# if they want.
+@_SpecialForm
+def Never(self, parameters):
+ """The bottom type, a type that has no members.
+
+ This can be used to define a function that should never be
+ called, or a function that never returns::
+
+ from typing import Never
+
+ def never_call_me(arg: Never) -> None:
+ pass
+
+ def int_or_str(arg: int | str) -> None:
+ never_call_me(arg) # type checker error
+ match arg:
+ case int():
+ print("It's an int")
+ case str():
+ print("It's a str")
+ case _:
+ never_call_me(arg) # ok, arg is of type Never
+ """
+ raise TypeError(f"{self} is not subscriptable")
+
+
+@_SpecialForm
+def Self(self, parameters):
+ """Used to spell the type of "self" in classes.
+
+ Example::
+
+ from typing import Self
+
+ class Foo:
+ def return_self(self) -> Self:
+ ...
+ return self
+
+ This is especially useful for:
+ - classmethods that are used as alternative constructors
+ - annotating an `__enter__` method which returns self
+ """
+ raise TypeError(f"{self} is not subscriptable")
+
+
+@_SpecialForm
+def LiteralString(self, parameters):
+ """Represents an arbitrary literal string.
+
+ Example::
+
+ from typing import LiteralString
+
+ def run_query(sql: LiteralString) -> ...
+ ...
+
+ def caller(arbitrary_string: str, literal_string: LiteralString) -> None:
+ run_query("SELECT * FROM students") # ok
+ run_query(literal_string) # ok
+ run_query("SELECT * FROM " + literal_string) # ok
+ run_query(arbitrary_string) # type checker error
+ run_query( # type checker error
+ f"SELECT * FROM students WHERE name = {arbitrary_string}"
+ )
+
+ Only string literals and other LiteralStrings are compatible
+ with LiteralString. This provides a tool to help prevent
+ security issues such as SQL injection.
+ """
+ raise TypeError(f"{self} is not subscriptable")
+
+
@_SpecialForm
def ClassVar(self, parameters):
"""Special type construct to mark class variables.
@@ -447,9 +616,9 @@
attribute is intended to be used as a class variable and
should not be set on instances of that class. Usage::
- class Starship:
- stats: ClassVar[Dict[str, int]] = {} # class variable
- damage: int = 10 # instance variable
+ class Starship:
+ stats: ClassVar[Dict[str, int]] = {} # class variable
+ damage: int = 10 # instance variable
ClassVar accepts only types and cannot be further subscribed.
@@ -464,16 +633,17 @@
"""Special typing construct to indicate final names to type checkers.
A final name cannot be re-assigned or overridden in a subclass.
- For example:
- MAX_SIZE: Final = 9000
- MAX_SIZE += 1 # Error reported by type checker
+ For example::
- class Connection:
- TIMEOUT: Final[int] = 10
+ MAX_SIZE: Final = 9000
+ MAX_SIZE += 1 # Error reported by type checker
- class FastConnector(Connection):
- TIMEOUT = 1 # Error reported by type checker
+ class Connection:
+ TIMEOUT: Final[int] = 10
+
+ class FastConnector(Connection):
+ TIMEOUT = 1 # Error reported by type checker
There is no runtime checking of these properties.
"""
@@ -484,25 +654,29 @@
def Union(self, parameters):
"""Union type; Union[X, Y] means either X or Y.
- To define a union, use e.g. Union[int, str]. Details:
+ On Python 3.10 and higher, the | operator
+ can also be used to denote unions;
+ X | Y means the same thing to the type checker as Union[X, Y].
+
+ To define a union, use e.g. Union[int, str]. Details:
- The arguments must be types and there must be at least one.
- None as an argument is a special case and is replaced by
type(None).
- Unions of unions are flattened, e.g.::
- Union[Union[int, str], float] == Union[int, str, float]
+ assert Union[Union[int, str], float] == Union[int, str, float]
- Unions of a single argument vanish, e.g.::
- Union[int] == int # The constructor actually returns int
+ assert Union[int] == int # The constructor actually returns int
- Redundant arguments are skipped, e.g.::
- Union[int, str, int] == Union[int, str]
+ assert Union[int, str, int] == Union[int, str]
- When comparing unions, the argument order is ignored, e.g.::
- Union[int, str] == Union[str, int]
+ assert Union[int, str] == Union[str, int]
- You cannot subclass or instantiate a union.
- You can use Optional[X] as a shorthand for Union[X, None].
@@ -522,10 +696,7 @@
@_SpecialForm
def Optional(self, parameters):
- """Optional type.
-
- Optional[X] is equivalent to Union[X, None].
- """
+ """Optional[X] is equivalent to Union[X, None]."""
arg = _type_check(parameters, f"{self} requires a single type.")
return Union[arg, type(None)]
@@ -536,17 +707,17 @@
This form can be used to indicate to type checkers that the corresponding
variable or function parameter has a value equivalent to the provided
- literal (or one of several literals):
+ literal (or one of several literals)::
- def validate_simple(data: Any) -> Literal[True]: # always returns True
- ...
+ def validate_simple(data: Any) -> Literal[True]: # always returns True
+ ...
- MODE = Literal['r', 'rb', 'w', 'wb']
- def open_helper(file: str, mode: MODE) -> str:
- ...
+ MODE = Literal['r', 'rb', 'w', 'wb']
+ def open_helper(file: str, mode: MODE) -> str:
+ ...
- open_helper('/some/path', 'r') # Passes type check
- open_helper('/other/path', 'typo') # Error in type checker
+ open_helper('/some/path', 'r') # Passes type check
+ open_helper('/other/path', 'typo') # Error in type checker
Literal[...] cannot be subclassed. At runtime, an arbitrary value
is allowed as type argument to Literal[...], but type checkers may
@@ -566,11 +737,11 @@
@_SpecialForm
def TypeAlias(self, parameters):
- """Special marker indicating that an assignment should
- be recognized as a proper type alias definition by type
- checkers.
+ """Special form for marking type aliases.
- For example::
+ Use TypeAlias to indicate that an assignment should
+ be recognized as a proper type alias definition by type
+ checkers. For example::
Predicate: TypeAlias = Callable[..., bool]
@@ -581,13 +752,15 @@
@_SpecialForm
def Concatenate(self, parameters):
- """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a
- higher order function which adds, removes or transforms parameters of a
- callable.
+ """Special form for annotating higher-order functions.
+
+ ``Concatenate`` can be sed in conjunction with ``ParamSpec`` and
+ ``Callable`` to represent a higher order function which adds, removes or
+ transforms the parameters of a callable.
For example::
- Callable[Concatenate[int, P], int]
+ Callable[Concatenate[int, P], int]
See PEP 612 for detailed information.
"""
@@ -595,19 +768,20 @@
raise TypeError("Cannot take a Concatenate of no types.")
if not isinstance(parameters, tuple):
parameters = (parameters,)
- if not isinstance(parameters[-1], ParamSpec):
+ if not (parameters[-1] is ... or isinstance(parameters[-1], ParamSpec)):
raise TypeError("The last parameter to Concatenate should be a "
- "ParamSpec variable.")
+ "ParamSpec variable or ellipsis.")
msg = "Concatenate[arg, ...]: each arg must be a type."
parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1])
return _ConcatenateGenericAlias(self, parameters,
- _typevar_types=(TypeVar, ParamSpec),
_paramspec_tvars=True)
@_SpecialForm
def TypeGuard(self, parameters):
- """Special typing form used to annotate the return type of a user-defined
+ """Special typing construct for marking user-defined type guard functions.
+
+ ``TypeGuard`` can be used to annotate the return type of a user-defined
type guard function. ``TypeGuard`` only accepts a single type argument.
At runtime, functions marked this way should return a boolean.
@@ -630,14 +804,14 @@
For example::
- def is_str(val: Union[str, float]):
- # "isinstance" type guard
- if isinstance(val, str):
- # Type of ``val`` is narrowed to ``str``
- ...
- else:
- # Else, type of ``val`` is narrowed to ``float``.
- ...
+ def is_str(val: Union[str, float]):
+ # "isinstance" type guard
+ if isinstance(val, str):
+ # Type of ``val`` is narrowed to ``str``
+ ...
+ else:
+ # Else, type of ``val`` is narrowed to ``float``.
+ ...
Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower
form of ``TypeA`` (it can even be a wider form) and this may lead to
@@ -664,10 +838,19 @@
def __init__(self, arg, is_argument=True, module=None, *, is_class=False):
if not isinstance(arg, str):
raise TypeError(f"Forward reference must be a string -- got {arg!r}")
+
+ # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`.
+ # Unfortunately, this isn't a valid expression on its own, so we
+ # do the unpacking manually.
+ if arg[0] == '*':
+ arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0]
+ else:
+ arg_to_compile = arg
try:
- code = compile(arg, '<string>', 'eval')
+ code = compile(arg_to_compile, '<string>', 'eval')
except SyntaxError:
raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}")
+
self.__forward_arg__ = arg
self.__forward_code__ = code
self.__forward_evaluated__ = False
@@ -714,11 +897,44 @@
def __hash__(self):
return hash((self.__forward_arg__, self.__forward_module__))
- def __repr__(self):
- return f'ForwardRef({self.__forward_arg__!r})'
+ def __or__(self, other):
+ return Union[self, other]
-class _TypeVarLike:
- """Mixin for TypeVar-like types (TypeVar and ParamSpec)."""
+ def __ror__(self, other):
+ return Union[other, self]
+
+ def __repr__(self):
+ if self.__forward_module__ is None:
+ module_repr = ''
+ else:
+ module_repr = f', module={self.__forward_module__!r}'
+ return f'ForwardRef({self.__forward_arg__!r}{module_repr})'
+
+
+def _is_unpacked_typevartuple(x: Any) -> bool:
+ return ((not isinstance(x, type)) and
+ getattr(x, '__typing_is_unpacked_typevartuple__', False))
+
+
+def _is_typevar_like(x: Any) -> bool:
+ return isinstance(x, (TypeVar, ParamSpec)) or _is_unpacked_typevartuple(x)
+
+
+class _PickleUsingNameMixin:
+ """Mixin enabling pickling based on self.__name__."""
+
+ def __reduce__(self):
+ return self.__name__
+
+
+class _BoundVarianceMixin:
+ """Mixin giving __init__ bound and variance arguments.
+
+ This is used by TypeVar and ParamSpec, which both employ the notions of
+ a type 'bound' (restricting type arguments to be a subtype of some
+ specified type) and type 'variance' (determining subtype relations between
+ generic types).
+ """
def __init__(self, bound, covariant, contravariant):
"""Used to setup TypeVars and ParamSpec's bound, covariant and
contravariant attributes.
@@ -747,11 +963,9 @@
prefix = '~'
return prefix + self.__name__
- def __reduce__(self):
- return self.__name__
-
-class TypeVar( _Final, _Immutable, _TypeVarLike, _root=True):
+class TypeVar(_Final, _Immutable, _BoundVarianceMixin, _PickleUsingNameMixin,
+ _root=True):
"""Type variable.
Usage::
@@ -795,9 +1009,6 @@
Note that only type variables defined in global scope can be pickled.
"""
- __slots__ = ('__name__', '__bound__', '__constraints__',
- '__covariant__', '__contravariant__', '__dict__')
-
def __init__(self, name, *constraints, bound=None,
covariant=False, contravariant=False):
self.__name__ = name
@@ -808,13 +1019,97 @@
raise TypeError("A single constraint is not allowed")
msg = "TypeVar(name, constraint, ...): constraints must be types."
self.__constraints__ = tuple(_type_check(t, msg) for t in constraints)
- try:
- def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling
- except (AttributeError, ValueError):
- def_mod = None
+ def_mod = _caller()
if def_mod != 'typing':
self.__module__ = def_mod
+ def __typing_subst__(self, arg):
+ msg = "Parameters to generic types must be types."
+ arg = _type_check(arg, msg, is_argument=True)
+ if ((isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack) or
+ (isinstance(arg, GenericAlias) and getattr(arg, '__unpacked__', False))):
+ raise TypeError(f"{arg} is not valid as type argument")
+ return arg
+
+
+class TypeVarTuple(_Final, _Immutable, _PickleUsingNameMixin, _root=True):
+ """Type variable tuple.
+
+ Usage:
+
+ Ts = TypeVarTuple('Ts') # Can be given any name
+
+ Just as a TypeVar (type variable) is a placeholder for a single type,
+ a TypeVarTuple is a placeholder for an *arbitrary* number of types. For
+ example, if we define a generic class using a TypeVarTuple:
+
+ class C(Generic[*Ts]): ...
+
+ Then we can parameterize that class with an arbitrary number of type
+ arguments:
+
+ C[int] # Fine
+ C[int, str] # Also fine
+ C[()] # Even this is fine
+
+ For more details, see PEP 646.
+
+ Note that only TypeVarTuples defined in global scope can be pickled.
+ """
+
+ def __init__(self, name):
+ self.__name__ = name
+
+ # Used for pickling.
+ def_mod = _caller()
+ if def_mod != 'typing':
+ self.__module__ = def_mod
+
+ def __iter__(self):
+ yield Unpack[self]
+
+ def __repr__(self):
+ return self.__name__
+
+ def __typing_subst__(self, arg):
+ raise TypeError("Substitution of bare TypeVarTuple is not supported")
+
+ def __typing_prepare_subst__(self, alias, args):
+ params = alias.__parameters__
+ typevartuple_index = params.index(self)
+ for param in params[typevartuple_index + 1:]:
+ if isinstance(param, TypeVarTuple):
+ raise TypeError(f"More than one TypeVarTuple parameter in {alias}")
+
+ alen = len(args)
+ plen = len(params)
+ left = typevartuple_index
+ right = plen - typevartuple_index - 1
+ var_tuple_index = None
+ fillarg = None
+ for k, arg in enumerate(args):
+ if not isinstance(arg, type):
+ subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
+ if subargs and len(subargs) == 2 and subargs[-1] is ...:
+ if var_tuple_index is not None:
+ raise TypeError("More than one unpacked arbitrary-length tuple argument")
+ var_tuple_index = k
+ fillarg = subargs[0]
+ if var_tuple_index is not None:
+ left = min(left, var_tuple_index)
+ right = min(right, alen - var_tuple_index - 1)
+ elif left + right > alen:
+ raise TypeError(f"Too few arguments for {alias};"
+ f" actual {alen}, expected at least {plen-1}")
+
+ return (
+ *args[:left],
+ *([fillarg]*(typevartuple_index - left)),
+ tuple(args[left: alen - right]),
+ *([fillarg]*(plen - right - left - typevartuple_index - 1)),
+ *args[alen - right:],
+ )
+
class ParamSpecArgs(_Final, _Immutable, _root=True):
"""The args for a ParamSpec object.
@@ -864,7 +1159,8 @@
return self.__origin__ == other.__origin__
-class ParamSpec(_Final, _Immutable, _TypeVarLike, _root=True):
+class ParamSpec(_Final, _Immutable, _BoundVarianceMixin, _PickleUsingNameMixin,
+ _root=True):
"""Parameter specification variable.
Usage::
@@ -894,25 +1190,14 @@
'''Add two numbers together.'''
return x + y
- Parameter specification variables defined with covariant=True or
- contravariant=True can be used to declare covariant or contravariant
- generic types. These keyword arguments are valid, but their actual semantics
- are yet to be decided. See PEP 612 for details.
-
Parameter specification variables can be introspected. e.g.:
P.__name__ == 'P'
- P.__bound__ == None
- P.__covariant__ == False
- P.__contravariant__ == False
Note that only parameter specification variables defined in global scope can
be pickled.
"""
- __slots__ = ('__name__', '__bound__', '__covariant__', '__contravariant__',
- '__dict__')
-
@property
def args(self):
return ParamSpecArgs(self)
@@ -924,26 +1209,45 @@
def __init__(self, name, *, bound=None, covariant=False, contravariant=False):
self.__name__ = name
super().__init__(bound, covariant, contravariant)
- try:
- def_mod = sys._getframe(1).f_globals.get('__name__', '__main__')
- except (AttributeError, ValueError):
- def_mod = None
+ def_mod = _caller()
if def_mod != 'typing':
self.__module__ = def_mod
+ def __typing_subst__(self, arg):
+ if isinstance(arg, (list, tuple)):
+ arg = tuple(_type_check(a, "Expected a type.") for a in arg)
+ elif not _is_param_expr(arg):
+ raise TypeError(f"Expected a list of types, an ellipsis, "
+ f"ParamSpec, or Concatenate. Got {arg}")
+ return arg
+
+ def __typing_prepare_subst__(self, alias, args):
+ params = alias.__parameters__
+ i = params.index(self)
+ if i >= len(args):
+ raise TypeError(f"Too few arguments for {alias}")
+ # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612.
+ if len(params) == 1 and not _is_param_expr(args[0]):
+ assert i == 0
+ args = (args,)
+ # Convert lists to tuples to help other libraries cache the results.
+ elif isinstance(args[i], list):
+ args = (*args[:i], tuple(args[i]), *args[i+1:])
+ return args
def _is_dunder(attr):
return attr.startswith('__') and attr.endswith('__')
class _BaseGenericAlias(_Final, _root=True):
- """The central part of internal API.
+ """The central part of the internal API.
This represents a generic version of type 'origin' with type arguments 'params'.
There are two kind of these aliases: user defined and special. The special ones
are wrappers around builtin collections and ABCs in collections.abc. These must
- have 'name' always set. If 'inst' is False, then the alias can't be instantiated,
+ have 'name' always set. If 'inst' is False, then the alias can't be instantiated;
this is used by e.g. typing.List and typing.Dict.
"""
+
def __init__(self, origin, *, inst=True, name=None):
self._inst = inst
self._name = name
@@ -985,7 +1289,7 @@
def __setattr__(self, attr, val):
if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams',
- '_typevar_types', '_paramspec_tvars'}:
+ '_paramspec_tvars'}:
super().__setattr__(attr, val)
else:
setattr(self.__origin__, attr, val)
@@ -1001,6 +1305,7 @@
return list(set(super().__dir__()
+ [attr for attr in dir(self.__origin__) if not _is_dunder(attr)]))
+
# Special typing constructs Union, Optional, Generic, Callable and Tuple
# use three special attributes for internal bookkeeping of generic types:
# * __parameters__ is a tuple of unique free type parameters of a generic
@@ -1013,17 +1318,39 @@
class _GenericAlias(_BaseGenericAlias, _root=True):
- def __init__(self, origin, params, *, inst=True, name=None,
- _typevar_types=TypeVar,
+ # The type of parameterized generics.
+ #
+ # That is, for example, `type(List[int])` is `_GenericAlias`.
+ #
+ # Objects which are instances of this class include:
+ # * Parameterized container types, e.g. `Tuple[int]`, `List[int]`.
+ # * Note that native container types, e.g. `tuple`, `list`, use
+ # `types.GenericAlias` instead.
+ # * Parameterized classes:
+ # T = TypeVar('T')
+ # class C(Generic[T]): pass
+ # # C[int] is a _GenericAlias
+ # * `Callable` aliases, generic `Callable` aliases, and
+ # parameterized `Callable` aliases:
+ # T = TypeVar('T')
+ # # _CallableGenericAlias inherits from _GenericAlias.
+ # A = Callable[[], None] # _CallableGenericAlias
+ # B = Callable[[T], None] # _CallableGenericAlias
+ # C = B[int] # _CallableGenericAlias
+ # * Parameterized `Final`, `ClassVar` and `TypeGuard`:
+ # # All _GenericAlias
+ # Final[int]
+ # ClassVar[float]
+ # TypeVar[bool]
+
+ def __init__(self, origin, args, *, inst=True, name=None,
_paramspec_tvars=False):
super().__init__(origin, inst=inst, name=name)
- if not isinstance(params, tuple):
- params = (params,)
+ if not isinstance(args, tuple):
+ args = (args,)
self.__args__ = tuple(... if a is _TypingEllipsis else
- () if a is _TypingEmpty else
- a for a in params)
- self.__parameters__ = _collect_type_vars(params, typevar_types=_typevar_types)
- self._typevar_types = _typevar_types
+ a for a in args)
+ self.__parameters__ = _collect_parameters(args)
self._paramspec_tvars = _paramspec_tvars
if not name:
self.__module__ = origin.__module__
@@ -1044,45 +1371,130 @@
return Union[left, self]
@_tp_cache
- def __getitem__(self, params):
+ def __getitem__(self, args):
+ # Parameterizes an already-parameterized object.
+ #
+ # For example, we arrive here doing something like:
+ # T1 = TypeVar('T1')
+ # T2 = TypeVar('T2')
+ # T3 = TypeVar('T3')
+ # class A(Generic[T1]): pass
+ # B = A[T2] # B is a _GenericAlias
+ # C = B[T3] # Invokes _GenericAlias.__getitem__
+ #
+ # We also arrive here when parameterizing a generic `Callable` alias:
+ # T = TypeVar('T')
+ # C = Callable[[T], None]
+ # C[int] # Invokes _GenericAlias.__getitem__
+
if self.__origin__ in (Generic, Protocol):
# Can't subscript Generic[...] or Protocol[...].
raise TypeError(f"Cannot subscript already-subscripted {self}")
- if not isinstance(params, tuple):
- params = (params,)
- params = tuple(_type_convert(p) for p in params)
- if (self._paramspec_tvars
- and any(isinstance(t, ParamSpec) for t in self.__parameters__)):
- params = _prepare_paramspec_params(self, params)
- else:
- _check_generic(self, params, len(self.__parameters__))
+ if not self.__parameters__:
+ raise TypeError(f"{self} is not a generic class")
- subst = dict(zip(self.__parameters__, params))
+ # Preprocess `args`.
+ if not isinstance(args, tuple):
+ args = (args,)
+ args = tuple(_type_convert(p) for p in args)
+ args = _unpack_args(args)
+ new_args = self._determine_new_args(args)
+ r = self.copy_with(new_args)
+ return r
+
+ def _determine_new_args(self, args):
+ # Determines new __args__ for __getitem__.
+ #
+ # For example, suppose we had:
+ # T1 = TypeVar('T1')
+ # T2 = TypeVar('T2')
+ # class A(Generic[T1, T2]): pass
+ # T3 = TypeVar('T3')
+ # B = A[int, T3]
+ # C = B[str]
+ # `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`.
+ # Unfortunately, this is harder than it looks, because if `T3` is
+ # anything more exotic than a plain `TypeVar`, we need to consider
+ # edge cases.
+
+ params = self.__parameters__
+ # In the example above, this would be {T3: str}
+ for param in params:
+ prepare = getattr(param, '__typing_prepare_subst__', None)
+ if prepare is not None:
+ args = prepare(self, args)
+ alen = len(args)
+ plen = len(params)
+ if alen != plen:
+ raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
+ f" actual {alen}, expected {plen}")
+ new_arg_by_param = dict(zip(params, args))
+ return tuple(self._make_substitution(self.__args__, new_arg_by_param))
+
+ def _make_substitution(self, args, new_arg_by_param):
+ """Create a list of new type arguments."""
new_args = []
- for arg in self.__args__:
- if isinstance(arg, self._typevar_types):
- if isinstance(arg, ParamSpec):
- arg = subst[arg]
- if not _is_param_expr(arg):
- raise TypeError(f"Expected a list of types, an ellipsis, "
- f"ParamSpec, or Concatenate. Got {arg}")
- else:
- arg = subst[arg]
- elif isinstance(arg, (_GenericAlias, GenericAlias, types.UnionType)):
- subparams = arg.__parameters__
- if subparams:
- subargs = tuple(subst[x] for x in subparams)
- arg = arg[subargs]
- # Required to flatten out the args for CallableGenericAlias
- if self.__origin__ == collections.abc.Callable and isinstance(arg, tuple):
- new_args.extend(arg)
- else:
- new_args.append(arg)
- return self.copy_with(tuple(new_args))
+ for old_arg in args:
+ if isinstance(old_arg, type):
+ new_args.append(old_arg)
+ continue
- def copy_with(self, params):
- return self.__class__(self.__origin__, params, name=self._name, inst=self._inst,
- _typevar_types=self._typevar_types,
+ substfunc = getattr(old_arg, '__typing_subst__', None)
+ if substfunc:
+ new_arg = substfunc(new_arg_by_param[old_arg])
+ else:
+ subparams = getattr(old_arg, '__parameters__', ())
+ if not subparams:
+ new_arg = old_arg
+ else:
+ subargs = []
+ for x in subparams:
+ if isinstance(x, TypeVarTuple):
+ subargs.extend(new_arg_by_param[x])
+ else:
+ subargs.append(new_arg_by_param[x])
+ new_arg = old_arg[tuple(subargs)]
+
+ if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple):
+ # Consider the following `Callable`.
+ # C = Callable[[int], str]
+ # Here, `C.__args__` should be (int, str) - NOT ([int], str).
+ # That means that if we had something like...
+ # P = ParamSpec('P')
+ # T = TypeVar('T')
+ # C = Callable[P, T]
+ # D = C[[int, str], float]
+ # ...we need to be careful; `new_args` should end up as
+ # `(int, str, float)` rather than `([int, str], float)`.
+ new_args.extend(new_arg)
+ elif _is_unpacked_typevartuple(old_arg):
+ # Consider the following `_GenericAlias`, `B`:
+ # class A(Generic[*Ts]): ...
+ # B = A[T, *Ts]
+ # If we then do:
+ # B[float, int, str]
+ # The `new_arg` corresponding to `T` will be `float`, and the
+ # `new_arg` corresponding to `*Ts` will be `(int, str)`. We
+ # should join all these types together in a flat list
+ # `(float, int, str)` - so again, we should `extend`.
+ new_args.extend(new_arg)
+ elif isinstance(old_arg, tuple):
+ # Corner case:
+ # P = ParamSpec('P')
+ # T = TypeVar('T')
+ # class Base(Generic[P]): ...
+ # Can be substituted like this:
+ # X = Base[[int, T]]
+ # In this case, `old_arg` will be a tuple:
+ new_args.append(
+ tuple(self._make_substitution(old_arg, new_arg_by_param)),
+ )
+ else:
+ new_args.append(new_arg)
+ return new_args
+
+ def copy_with(self, args):
+ return self.__class__(self.__origin__, args, name=self._name, inst=self._inst,
_paramspec_tvars=self._paramspec_tvars)
def __repr__(self):
@@ -1090,7 +1502,11 @@
name = 'typing.' + self._name
else:
name = _type_repr(self.__origin__)
- args = ", ".join([_type_repr(a) for a in self.__args__])
+ if self.__args__:
+ args = ", ".join([_type_repr(a) for a in self.__args__])
+ else:
+ # To ensure the repr is eval-able.
+ args = "()"
return f'{name}[{args}]'
def __reduce__(self):
@@ -1099,8 +1515,7 @@
else:
origin = self.__origin__
args = tuple(self.__args__)
- if len(args) == 1 and (not isinstance(args[0], tuple) or
- origin is Tuple and not args[0]):
+ if len(args) == 1 and not isinstance(args[0], tuple):
args, = args
return operator.getitem, (origin, args)
@@ -1119,12 +1534,15 @@
return ()
return (self.__origin__,)
+ def __iter__(self):
+ yield Unpack[self]
+
# _nparams is the number of accepted parameters, e.g. 0 for Hashable,
# 1 for List and 2 for Dict. It may be -1 if variable number of
# parameters are accepted (needs custom __getitem__).
-class _SpecialGenericAlias(_BaseGenericAlias, _root=True):
+class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True):
def __init__(self, origin, nparams, *, inst=True, name=None):
if name is None:
name = origin.__name__
@@ -1167,7 +1585,7 @@
def __ror__(self, left):
return Union[left, self]
-class _CallableGenericAlias(_GenericAlias, _root=True):
+class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True):
def __repr__(self):
assert self._name == 'Callable'
args = self.__args__
@@ -1188,7 +1606,6 @@
def copy_with(self, params):
return _CallableGenericAlias(self.__origin__, params,
name=self._name, inst=self._inst,
- _typevar_types=(TypeVar, ParamSpec),
_paramspec_tvars=True)
def __getitem__(self, params):
@@ -1222,20 +1639,18 @@
class _TupleType(_SpecialGenericAlias, _root=True):
@_tp_cache
def __getitem__(self, params):
- if params == ():
- return self.copy_with((_TypingEmpty,))
if not isinstance(params, tuple):
params = (params,)
- if len(params) == 2 and params[1] is ...:
+ if len(params) >= 2 and params[-1] is ...:
msg = "Tuple[t, ...]: t must be a type."
- p = _type_check(params[0], msg)
- return self.copy_with((p, _TypingEllipsis))
+ params = tuple(_type_check(p, msg) for p in params[:-1])
+ return self.copy_with((*params, _TypingEllipsis))
msg = "Tuple[t0, t1, ...]: each t must be a type."
params = tuple(_type_check(p, msg) for p in params)
return self.copy_with(params)
-class _UnionGenericAlias(_GenericAlias, _root=True):
+class _UnionGenericAlias(_NotIterable, _GenericAlias, _root=True):
def copy_with(self, params):
return Union[params]
@@ -1274,7 +1689,6 @@
class _LiteralGenericAlias(_GenericAlias, _root=True):
-
def __eq__(self, other):
if not isinstance(other, _LiteralGenericAlias):
return NotImplemented
@@ -1291,12 +1705,71 @@
return (*params[:-1], *params[-1])
if isinstance(params[-1], _ConcatenateGenericAlias):
params = (*params[:-1], *params[-1].__args__)
- elif not isinstance(params[-1], ParamSpec):
- raise TypeError("The last parameter to Concatenate should be a "
- "ParamSpec variable.")
return super().copy_with(params)
+@_SpecialForm
+def Unpack(self, parameters):
+ """Type unpack operator.
+
+ The type unpack operator takes the child types from some container type,
+ such as `tuple[int, str]` or a `TypeVarTuple`, and 'pulls them out'. For
+ example::
+
+ # For some generic class `Foo`:
+ Foo[Unpack[tuple[int, str]]] # Equivalent to Foo[int, str]
+
+ Ts = TypeVarTuple('Ts')
+ # Specifies that `Bar` is generic in an arbitrary number of types.
+ # (Think of `Ts` as a tuple of an arbitrary number of individual
+ # `TypeVar`s, which the `Unpack` is 'pulling out' directly into the
+ # `Generic[]`.)
+ class Bar(Generic[Unpack[Ts]]): ...
+ Bar[int] # Valid
+ Bar[int, str] # Also valid
+
+ From Python 3.11, this can also be done using the `*` operator::
+
+ Foo[*tuple[int, str]]
+ class Bar(Generic[*Ts]): ...
+
+ Note that there is only some runtime checking of this operator. Not
+ everything the runtime allows may be accepted by static type checkers.
+
+ For more information, see PEP 646.
+ """
+ item = _type_check(parameters, f'{self} accepts only single type.')
+ return _UnpackGenericAlias(origin=self, args=(item,))
+
+
+class _UnpackGenericAlias(_GenericAlias, _root=True):
+ def __repr__(self):
+ # `Unpack` only takes one argument, so __args__ should contain only
+ # a single item.
+ return '*' + repr(self.__args__[0])
+
+ def __getitem__(self, args):
+ if self.__typing_is_unpacked_typevartuple__:
+ return args
+ return super().__getitem__(args)
+
+ @property
+ def __typing_unpacked_tuple_args__(self):
+ assert self.__origin__ is Unpack
+ assert len(self.__args__) == 1
+ arg, = self.__args__
+ if isinstance(arg, _GenericAlias):
+ assert arg.__origin__ is tuple
+ return arg.__args__
+ return None
+
+ @property
+ def __typing_is_unpacked_typevartuple__(self):
+ assert self.__origin__ is Unpack
+ assert len(self.__args__) == 1
+ return isinstance(self.__args__[0], TypeVarTuple)
+
+
class Generic:
"""Abstract base class for generic types.
@@ -1322,15 +1795,26 @@
@_tp_cache
def __class_getitem__(cls, params):
+ """Parameterizes a generic class.
+
+ At least, parameterizing a generic class is the *main* thing this method
+ does. For example, for some generic class `Foo`, this is called when we
+ do `Foo[int]` - there, with `cls=Foo` and `params=int`.
+
+ However, note that this method is also called when defining generic
+ classes in the first place with `class Foo(Generic[T]): ...`.
+ """
if not isinstance(params, tuple):
params = (params,)
- if not params and cls is not Tuple:
- raise TypeError(
- f"Parameter list to {cls.__qualname__}[...] cannot be empty")
+
params = tuple(_type_convert(p) for p in params)
if cls in (Generic, Protocol):
# Generic and Protocol can only be subscripted with unique type variables.
- if not all(isinstance(p, (TypeVar, ParamSpec)) for p in params):
+ if not params:
+ raise TypeError(
+ f"Parameter list to {cls.__qualname__}[...] cannot be empty"
+ )
+ if not all(_is_typevar_like(p) for p in params):
raise TypeError(
f"Parameters to {cls.__name__}[...] must all be type variables "
f"or parameter specification variables.")
@@ -1339,12 +1823,21 @@
f"Parameters to {cls.__name__}[...] must all be unique")
else:
# Subscripting a regular Generic subclass.
- if any(isinstance(t, ParamSpec) for t in cls.__parameters__):
- params = _prepare_paramspec_params(cls, params)
- else:
- _check_generic(cls, params, len(cls.__parameters__))
+ for param in cls.__parameters__:
+ prepare = getattr(param, '__typing_prepare_subst__', None)
+ if prepare is not None:
+ params = prepare(cls, params)
+ _check_generic(cls, params, len(cls.__parameters__))
+
+ new_args = []
+ for param, new_arg in zip(cls.__parameters__, params):
+ if isinstance(param, TypeVarTuple):
+ new_args.extend(new_arg)
+ else:
+ new_args.append(new_arg)
+ params = tuple(new_args)
+
return _GenericAlias(cls, params,
- _typevar_types=(TypeVar, ParamSpec),
_paramspec_tvars=True)
def __init_subclass__(cls, *args, **kwargs):
@@ -1353,11 +1846,13 @@
if '__orig_bases__' in cls.__dict__:
error = Generic in cls.__orig_bases__
else:
- error = Generic in cls.__bases__ and cls.__name__ != 'Protocol'
+ error = (Generic in cls.__bases__ and
+ cls.__name__ != 'Protocol' and
+ type(cls) != _TypedDictMeta)
if error:
raise TypeError("Cannot inherit from plain Generic")
if '__orig_bases__' in cls.__dict__:
- tvars = _collect_type_vars(cls.__orig_bases__, (TypeVar, ParamSpec))
+ tvars = _collect_parameters(cls.__orig_bases__)
# Look for Generic[T1, ..., Tn].
# If found, tvars must be a subset of it.
# If not found, tvars is it.
@@ -1369,7 +1864,7 @@
base.__origin__ is Generic):
if gvars is not None:
raise TypeError(
- "Cannot inherit from Generic[...] multiple types.")
+ "Cannot inherit from Generic[...] multiple times.")
gvars = base.__parameters__
if gvars is not None:
tvarset = set(tvars)
@@ -1383,13 +1878,6 @@
cls.__parameters__ = tuple(tvars)
-class _TypingEmpty:
- """Internal placeholder for () or []. Used by TupleMeta and CallableMeta
- to allow empty list/tuple in specific places, without allowing them
- to sneak in where prohibited.
- """
-
-
class _TypingEllipsis:
"""Internal placeholder for ... (ellipsis)."""
@@ -1469,10 +1957,7 @@
The abc and functools modules indiscriminately call isinstance() and
issubclass() on the whole MRO of a user class, which may contain protocols.
"""
- try:
- return sys._getframe(depth).f_globals['__name__'] in ['abc', 'functools']
- except (AttributeError, ValueError): # For platforms without _getframe().
- return True
+ return _caller(depth) in {'abc', 'functools', None}
_PROTO_ALLOWLIST = {
@@ -1542,6 +2027,7 @@
def meth(self) -> T:
...
"""
+
__slots__ = ()
_is_protocol = True
_is_runtime_protocol = False
@@ -1607,17 +2093,21 @@
issubclass(base, Generic) and base._is_protocol):
raise TypeError('Protocols can only inherit from other'
' protocols, got %r' % base)
- cls.__init__ = _no_init_or_replace_init
+ if cls.__init__ is Protocol.__init__:
+ cls.__init__ = _no_init_or_replace_init
-class _AnnotatedAlias(_GenericAlias, _root=True):
+class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True):
"""Runtime representation of an annotated type.
At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't'
- with extra annotations. The alias behaves like a normal typing alias,
- instantiating is the same as instantiating the underlying type, binding
+ with extra annotations. The alias behaves like a normal typing alias.
+ Instantiating is the same as instantiating the underlying type; binding
it to types is also the same.
+
+ The metadata itself is stored in a '__metadata__' attribute as a tuple.
"""
+
def __init__(self, origin, metadata):
if isinstance(origin, _AnnotatedAlias):
metadata = origin.__metadata__ + metadata
@@ -1657,7 +2147,7 @@
class Annotated:
- """Add context specific metadata to a type.
+ """Add context-specific metadata to a type.
Example: Annotated[int, runtime_check.Unsigned] indicates to the
hypothetical runtime_check module that this type is an unsigned int.
@@ -1669,22 +2159,37 @@
Details:
- It's an error to call `Annotated` with less than two arguments.
+ - Access the metadata via the ``__metadata__`` attribute::
+
+ assert Annotated[int, '$'].__metadata__ == ('$',)
+
- Nested Annotated are flattened::
- Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3]
+ assert Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3]
- Instantiating an annotated type is equivalent to instantiating the
underlying type::
- Annotated[C, Ann1](5) == C(5)
+ assert Annotated[C, Ann1](5) == C(5)
- Annotated can be used as a generic type alias::
Optimized = Annotated[T, runtime.Optimize()]
- Optimized[int] == Annotated[int, runtime.Optimize()]
+ assert Optimized[int] == Annotated[int, runtime.Optimize()]
OptimizedList = Annotated[List[T], runtime.Optimize()]
- OptimizedList[int] == Annotated[List[int], runtime.Optimize()]
+ assert OptimizedList[int] == Annotated[List[int], runtime.Optimize()]
+
+ - Annotated cannot be used with an unpacked TypeVarTuple::
+
+ Annotated[*Ts, Ann1] # NOT valid
+
+ This would be equivalent to
+
+ Annotated[T1, T2, T3, ..., Ann1]
+
+ where T1, T2 etc. are TypeVars, which would be invalid, because
+ only one type should be passed to Annotated.
"""
__slots__ = ()
@@ -1698,6 +2203,9 @@
raise TypeError("Annotated[...] should be used "
"with at least two arguments (a type and an "
"annotation).")
+ if _is_unpacked_typevartuple(params[0]):
+ raise TypeError("Annotated[...] should not be used with an "
+ "unpacked TypeVarTuple")
msg = "Annotated[t, ...]: t must be a type."
origin = _type_check(params[0], msg, allow_special_forms=True)
metadata = tuple(params[1:])
@@ -1716,6 +2224,7 @@
Raise TypeError if applied to a non-protocol class.
This allows a simple-minded structural check very similar to
one trick ponies in collections.abc such as Iterable.
+
For example::
@runtime_checkable
@@ -1745,24 +2254,20 @@
return val
-def _get_defaults(func):
- """Internal helper to extract the default arguments, by name."""
- try:
- code = func.__code__
- except AttributeError:
- # Some built-in functions don't have __code__, __defaults__, etc.
- return {}
- pos_count = code.co_argcount
- arg_names = code.co_varnames
- arg_names = arg_names[:pos_count]
- defaults = func.__defaults__ or ()
- kwdefaults = func.__kwdefaults__
- res = dict(kwdefaults) if kwdefaults else {}
- pos_offset = pos_count - len(defaults)
- for name, value in zip(arg_names[pos_offset:], defaults):
- assert name not in res
- res[name] = value
- return res
+def assert_type(val, typ, /):
+ """Ask a static type checker to confirm that the value is of the given type.
+
+ At runtime this does nothing: it returns the first argument unchanged with no
+ checks or side effects, no matter the actual type of the argument.
+
+ When a static type checker encounters a call to assert_type(), it
+ emits an error if the value is not of the specified type::
+
+ def greet(name: str) -> None:
+ assert_type(name, str) # ok
+ assert_type(name, int) # type checker error
+ """
+ return val
_allowed_types = (types.FunctionType, types.BuiltinFunctionType,
@@ -1774,8 +2279,7 @@
"""Return type hints for an object.
This is often the same as obj.__annotations__, but it handles
- forward references encoded as string literals, adds Optional[t] if a
- default value equal to None is set and recursively replaces all
+ forward references encoded as string literals and recursively replaces all
'Annotated[T, ...]' with 'T' (unless 'include_extras=True').
The argument may be a module, class, method, or function. The annotations
@@ -1802,7 +2306,6 @@
- If two dict arguments are passed, they specify globals and
locals, respectively.
"""
-
if getattr(obj, '__no_type_check__', None):
return {}
# Classes require a special treatment.
@@ -1855,7 +2358,6 @@
else:
raise TypeError('{!r} is not a module, class, method, '
'or function.'.format(obj))
- defaults = _get_defaults(obj)
hints = dict(hints)
for name, value in hints.items():
if value is None:
@@ -1868,18 +2370,16 @@
is_argument=not isinstance(obj, types.ModuleType),
is_class=False,
)
- value = _eval_type(value, globalns, localns)
- if name in defaults and defaults[name] is None:
- value = Optional[value]
- hints[name] = value
+ hints[name] = _eval_type(value, globalns, localns)
return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()}
def _strip_annotations(t):
- """Strips the annotations from a given type.
- """
+ """Strip the annotations from a given type."""
if isinstance(t, _AnnotatedAlias):
return _strip_annotations(t.__origin__)
+ if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired):
+ return _strip_annotations(t.__args__[0])
if isinstance(t, _GenericAlias):
stripped_args = tuple(_strip_annotations(a) for a in t.__args__)
if stripped_args == t.__args__:
@@ -1903,16 +2403,16 @@
"""Get the unsubscripted version of a type.
This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar
- and Annotated. Return None for unsupported types. Examples::
+ Annotated, and others. Return None for unsupported types. Examples::
- get_origin(Literal[42]) is Literal
- get_origin(int) is None
- get_origin(ClassVar[int]) is ClassVar
- get_origin(Generic) is Generic
- get_origin(Generic[T]) is Generic
- get_origin(Union[T, int]) is Union
- get_origin(List[Tuple[T, T]][int]) == list
- get_origin(P.args) is P
+ assert get_origin(Literal[42]) is Literal
+ assert get_origin(int) is None
+ assert get_origin(ClassVar[int]) is ClassVar
+ assert get_origin(Generic) is Generic
+ assert get_origin(Generic[T]) is Generic
+ assert get_origin(Union[T, int]) is Union
+ assert get_origin(List[Tuple[T, T]][int]) is list
+ assert get_origin(P.args) is P
"""
if isinstance(tp, _AnnotatedAlias):
return Annotated
@@ -1930,19 +2430,20 @@
"""Get type arguments with all substitutions performed.
For unions, basic simplifications used by Union constructor are performed.
+
Examples::
- get_args(Dict[str, int]) == (str, int)
- get_args(int) == ()
- get_args(Union[int, Union[T, int], str][int]) == (int, str)
- get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
- get_args(Callable[[], T][int]) == ([], int)
+
+ assert get_args(Dict[str, int]) == (str, int)
+ assert get_args(int) == ()
+ assert get_args(Union[int, Union[T, int], str][int]) == (int, str)
+ assert get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
+ assert get_args(Callable[[], T][int]) == ([], int)
"""
if isinstance(tp, _AnnotatedAlias):
return (tp.__origin__,) + tp.__metadata__
if isinstance(tp, (_GenericAlias, GenericAlias)):
res = tp.__args__
- if (tp.__origin__ is collections.abc.Callable
- and not (len(res) == 2 and _is_param_expr(res[0]))):
+ if _should_unflatten_callable_args(tp, res):
res = (list(res[:-1]), res[-1])
return res
if isinstance(tp, types.UnionType):
@@ -1951,19 +2452,48 @@
def is_typeddict(tp):
- """Check if an annotation is a TypedDict class
+ """Check if an annotation is a TypedDict class.
For example::
+
class Film(TypedDict):
title: str
year: int
- is_typeddict(Film) # => True
+ is_typeddict(Film) # => True
is_typeddict(Union[list, str]) # => False
"""
return isinstance(tp, _TypedDictMeta)
+_ASSERT_NEVER_REPR_MAX_LENGTH = 100
+
+
+def assert_never(arg: Never, /) -> Never:
+ """Statically assert that a line of code is unreachable.
+
+ Example::
+
+ def int_or_str(arg: int | str) -> None:
+ match arg:
+ case int():
+ print("It's an int")
+ case str():
+ print("It's a str")
+ case _:
+ assert_never(arg)
+
+ If a type checker finds that a call to assert_never() is
+ reachable, it will emit an error.
+
+ At runtime, this throws an exception when called.
+ """
+ value = repr(arg)
+ if len(value) > _ASSERT_NEVER_REPR_MAX_LENGTH:
+ value = value[:_ASSERT_NEVER_REPR_MAX_LENGTH] + '...'
+ raise AssertionError(f"Expected code to be unreachable, but got: {value}")
+
+
def no_type_check(arg):
"""Decorator to indicate that annotations are not type hints.
@@ -1974,13 +2504,23 @@
This mutates the function(s) or class(es) in place.
"""
if isinstance(arg, type):
- arg_attrs = arg.__dict__.copy()
- for attr, val in arg.__dict__.items():
- if val in arg.__bases__ + (arg,):
- arg_attrs.pop(attr)
- for obj in arg_attrs.values():
+ for key in dir(arg):
+ obj = getattr(arg, key)
+ if (
+ not hasattr(obj, '__qualname__')
+ or obj.__qualname__ != f'{arg.__qualname__}.{obj.__name__}'
+ or getattr(obj, '__module__', None) != arg.__module__
+ ):
+ # We only modify objects that are defined in this type directly.
+ # If classes / methods are nested in multiple layers,
+ # we will modify them when processing their direct holders.
+ continue
+ # Instance, class, and static methods:
if isinstance(obj, types.FunctionType):
obj.__no_type_check__ = True
+ if isinstance(obj, types.MethodType):
+ obj.__func__.__no_type_check__ = True
+ # Nested types:
if isinstance(obj, type):
no_type_check(obj)
try:
@@ -1996,7 +2536,6 @@
This wraps the decorator with something that wraps the decorated
function in @no_type_check.
"""
-
@functools.wraps(decorator)
def wrapped_decorator(*args, **kwds):
func = decorator(*args, **kwds)
@@ -2015,58 +2554,99 @@
"by an implementation that is not @overload-ed.")
+# {module: {qualname: {firstlineno: func}}}
+_overload_registry = defaultdict(functools.partial(defaultdict, dict))
+
+
def overload(func):
"""Decorator for overloaded functions/methods.
In a stub file, place two or more stub definitions for the same
- function in a row, each decorated with @overload. For example:
+ function in a row, each decorated with @overload. For example::
- @overload
- def utf8(value: None) -> None: ...
- @overload
- def utf8(value: bytes) -> bytes: ...
- @overload
- def utf8(value: str) -> bytes: ...
+ @overload
+ def utf8(value: None) -> None: ...
+ @overload
+ def utf8(value: bytes) -> bytes: ...
+ @overload
+ def utf8(value: str) -> bytes: ...
In a non-stub file (i.e. a regular .py file), do the same but
follow it with an implementation. The implementation should *not*
- be decorated with @overload. For example:
+ be decorated with @overload. For example::
- @overload
- def utf8(value: None) -> None: ...
- @overload
- def utf8(value: bytes) -> bytes: ...
- @overload
- def utf8(value: str) -> bytes: ...
- def utf8(value):
- # implementation goes here
+ @overload
+ def utf8(value: None) -> None: ...
+ @overload
+ def utf8(value: bytes) -> bytes: ...
+ @overload
+ def utf8(value: str) -> bytes: ...
+ def utf8(value):
+ ... # implementation goes here
+
+ The overloads for a function can be retrieved at runtime using the
+ get_overloads() function.
"""
+ # classmethod and staticmethod
+ f = getattr(func, "__func__", func)
+ try:
+ _overload_registry[f.__module__][f.__qualname__][f.__code__.co_firstlineno] = func
+ except AttributeError:
+ # Not a normal function; ignore.
+ pass
return _overload_dummy
+def get_overloads(func):
+ """Return all defined overloads for *func* as a sequence."""
+ # classmethod and staticmethod
+ f = getattr(func, "__func__", func)
+ if f.__module__ not in _overload_registry:
+ return []
+ mod_dict = _overload_registry[f.__module__]
+ if f.__qualname__ not in mod_dict:
+ return []
+ return list(mod_dict[f.__qualname__].values())
+
+
+def clear_overloads():
+ """Clear all overloads in the registry."""
+ _overload_registry.clear()
+
+
def final(f):
- """A decorator to indicate final methods and final classes.
+ """Decorator to indicate final methods and final classes.
Use this decorator to indicate to type checkers that the decorated
method cannot be overridden, and decorated class cannot be subclassed.
- For example:
- class Base:
- @final
- def done(self) -> None:
- ...
- class Sub(Base):
- def done(self) -> None: # Error reported by type checker
+ For example::
+
+ class Base:
+ @final
+ def done(self) -> None:
+ ...
+ class Sub(Base):
+ def done(self) -> None: # Error reported by type checker
...
- @final
- class Leaf:
- ...
- class Other(Leaf): # Error reported by type checker
- ...
+ @final
+ class Leaf:
+ ...
+ class Other(Leaf): # Error reported by type checker
+ ...
- There is no runtime checking of these properties.
+ There is no runtime checking of these properties. The decorator
+ attempts to set the ``__final__`` attribute to ``True`` on the decorated
+ object to allow runtime introspection.
"""
+ try:
+ f.__final__ = True
+ except (AttributeError, TypeError):
+ # Skip the attribute silently if it is not writable.
+ # AttributeError happens if the object has __slots__ or a
+ # read-only property, TypeError if it's a builtin class.
+ pass
return f
@@ -2103,13 +2683,15 @@
Collection = _alias(collections.abc.Collection, 1)
Callable = _CallableType(collections.abc.Callable, 2)
Callable.__doc__ = \
- """Callable type; Callable[[int], str] is a function of (int) -> str.
+ """Deprecated alias to collections.abc.Callable.
+ Callable[[int], str] signifies a function of (int) -> str.
The subscription syntax must always be used with exactly two
- values: the argument list and the return type. The argument list
- must be a list of types or ellipsis; the return type must be a single type.
+ values: the argument list and the return type.
+ The argument list must be a list of types, a ParamSpec or ellipsis.
+ The return type must be a single type.
- There is no syntax to indicate optional or keyword arguments,
+ There is no syntax to indicate optional or keyword arguments;
such function types are rarely used as callback types.
"""
AbstractSet = _alias(collections.abc.Set, 1, name='AbstractSet')
@@ -2123,7 +2705,9 @@
# Tuple accepts variable number of parameters.
Tuple = _TupleType(tuple, -1, inst=False, name='Tuple')
Tuple.__doc__ = \
- """Tuple type; Tuple[X, Y] is the cross-product type of X and Y.
+ """Deprecated alias to builtins.tuple.
+
+ Tuple[X, Y] is the cross-product type of X and Y.
Example: Tuple[T1, T2] is a tuple of two elements corresponding
to type variables T1 and T2. Tuple[int, float, str] is a tuple
@@ -2150,25 +2734,26 @@
AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2)
Type = _alias(type, 1, inst=False, name='Type')
Type.__doc__ = \
- """A special construct usable to annotate class objects.
+ """Deprecated alias to builtins.type.
+ builtins.type or typing.Type can be used to annotate class objects.
For example, suppose we have the following classes::
- class User: ... # Abstract base for User classes
- class BasicUser(User): ...
- class ProUser(User): ...
- class TeamUser(User): ...
+ class User: ... # Abstract base for User classes
+ class BasicUser(User): ...
+ class ProUser(User): ...
+ class TeamUser(User): ...
And a function that takes a class argument that's a subclass of
User and returns an instance of the corresponding class::
- U = TypeVar('U', bound=User)
- def new_user(user_class: Type[U]) -> U:
- user = user_class()
- # (Here we could write the user object to a database)
- return user
+ U = TypeVar('U', bound=User)
+ def new_user(user_class: Type[U]) -> U:
+ user = user_class()
+ # (Here we could write the user object to a database)
+ return user
- joe = new_user(BasicUser)
+ joe = new_user(BasicUser)
At this point the type checker knows that joe has type BasicUser.
"""
@@ -2177,6 +2762,7 @@
@runtime_checkable
class SupportsInt(Protocol):
"""An ABC with one abstract method __int__."""
+
__slots__ = ()
@abstractmethod
@@ -2187,6 +2773,7 @@
@runtime_checkable
class SupportsFloat(Protocol):
"""An ABC with one abstract method __float__."""
+
__slots__ = ()
@abstractmethod
@@ -2197,6 +2784,7 @@
@runtime_checkable
class SupportsComplex(Protocol):
"""An ABC with one abstract method __complex__."""
+
__slots__ = ()
@abstractmethod
@@ -2207,6 +2795,7 @@
@runtime_checkable
class SupportsBytes(Protocol):
"""An ABC with one abstract method __bytes__."""
+
__slots__ = ()
@abstractmethod
@@ -2217,6 +2806,7 @@
@runtime_checkable
class SupportsIndex(Protocol):
"""An ABC with one abstract method __index__."""
+
__slots__ = ()
@abstractmethod
@@ -2227,6 +2817,7 @@
@runtime_checkable
class SupportsAbs(Protocol[T_co]):
"""An ABC with one abstract method __abs__ that is covariant in its return type."""
+
__slots__ = ()
@abstractmethod
@@ -2237,6 +2828,7 @@
@runtime_checkable
class SupportsRound(Protocol[T_co]):
"""An ABC with one abstract method __round__ that is covariant in its return type."""
+
__slots__ = ()
@abstractmethod
@@ -2263,9 +2855,13 @@
class NamedTupleMeta(type):
-
def __new__(cls, typename, bases, ns):
- assert bases[0] is _NamedTuple
+ assert _NamedTuple in bases
+ for base in bases:
+ if base is not _NamedTuple and base is not Generic:
+ raise TypeError(
+ 'can only inherit from a NamedTuple type and Generic')
+ bases = tuple(tuple if base is _NamedTuple else base for base in bases)
types = ns.get('__annotations__', {})
default_names = []
for field_name in types:
@@ -2279,19 +2875,25 @@
nm_tpl = _make_nmtuple(typename, types.items(),
defaults=[ns[n] for n in default_names],
module=ns['__module__'])
+ nm_tpl.__bases__ = bases
+ if Generic in bases:
+ class_getitem = Generic.__class_getitem__.__func__
+ nm_tpl.__class_getitem__ = classmethod(class_getitem)
# update from user namespace without overriding special namedtuple attributes
for key in ns:
if key in _prohibited:
raise AttributeError("Cannot overwrite NamedTuple attribute " + key)
elif key not in _special and key not in nm_tpl._fields:
setattr(nm_tpl, key, ns[key])
+ if Generic in bases:
+ nm_tpl.__init_subclass__()
return nm_tpl
def NamedTuple(typename, fields=None, /, **kwargs):
"""Typed version of namedtuple.
- Usage in Python versions >= 3.6::
+ Usage::
class Employee(NamedTuple):
name: str
@@ -2304,11 +2906,7 @@
The resulting class has an extra __annotations__ attribute, giving a
dict that maps field names to types. (The field names are also in
the _fields attribute, which is part of the namedtuple API.)
- Alternative equivalent keyword syntax is also accepted::
-
- Employee = NamedTuple('Employee', name=str, id=int)
-
- In Python versions <= 3.5 use::
+ An alternative equivalent functional syntax is also accepted::
Employee = NamedTuple('Employee', [('name', str), ('id', int)])
"""
@@ -2317,18 +2915,12 @@
elif kwargs:
raise TypeError("Either list of fields or keywords"
" can be provided to NamedTuple, not both")
- try:
- module = sys._getframe(1).f_globals.get('__name__', '__main__')
- except (AttributeError, ValueError):
- module = None
- return _make_nmtuple(typename, fields, module=module)
+ return _make_nmtuple(typename, fields, module=_caller())
_NamedTuple = type.__new__(NamedTupleMeta, 'NamedTuple', (), {})
def _namedtuple_mro_entries(bases):
- if len(bases) > 1:
- raise TypeError("Multiple inheritance with NamedTuple is not supported")
- assert bases[0] is NamedTuple
+ assert NamedTuple in bases
return (_NamedTuple,)
NamedTuple.__mro_entries__ = _namedtuple_mro_entries
@@ -2336,7 +2928,7 @@
class _TypedDictMeta(type):
def __new__(cls, name, bases, ns, total=True):
- """Create new typed dict class object.
+ """Create a new typed dict class object.
This method is called when TypedDict is subclassed,
or when TypedDict is instantiated. This way
@@ -2344,14 +2936,19 @@
Subclasses and instances of TypedDict return actual dictionaries.
"""
for base in bases:
- if type(base) is not _TypedDictMeta:
+ if type(base) is not _TypedDictMeta and base is not Generic:
raise TypeError('cannot inherit from both a TypedDict type '
'and a non-TypedDict base class')
- tp_dict = type.__new__(_TypedDictMeta, name, (dict,), ns)
+
+ if any(issubclass(b, Generic) for b in bases):
+ generic_base = (Generic,)
+ else:
+ generic_base = ()
+
+ tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns)
annotations = {}
own_annotations = ns.get('__annotations__', {})
- own_annotation_keys = set(own_annotations.keys())
msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type"
own_annotations = {
n: _type_check(tp, msg, module=tp_dict.__module__)
@@ -2366,10 +2963,22 @@
optional_keys.update(base.__dict__.get('__optional_keys__', ()))
annotations.update(own_annotations)
- if total:
- required_keys.update(own_annotation_keys)
- else:
- optional_keys.update(own_annotation_keys)
+ for annotation_key, annotation_type in own_annotations.items():
+ annotation_origin = get_origin(annotation_type)
+ if annotation_origin is Annotated:
+ annotation_args = get_args(annotation_type)
+ if annotation_args:
+ annotation_type = annotation_args[0]
+ annotation_origin = get_origin(annotation_type)
+
+ if annotation_origin is Required:
+ required_keys.add(annotation_key)
+ elif annotation_origin is NotRequired:
+ optional_keys.add(annotation_key)
+ elif total:
+ required_keys.add(annotation_key)
+ else:
+ optional_keys.add(annotation_key)
tp_dict.__annotations__ = annotations
tp_dict.__required_keys__ = frozenset(required_keys)
@@ -2390,10 +2999,11 @@
def TypedDict(typename, fields=None, /, *, total=True, **kwargs):
"""A simple typed namespace. At runtime it is equivalent to a plain dict.
- TypedDict creates a dictionary type that expects all of its
+ TypedDict creates a dictionary type such that a type checker will expect all
instances to have a certain set of keys, where each key is
associated with a value of a consistent type. This expectation
- is not checked at runtime but is only enforced by type checkers.
+ is not checked at runtime.
+
Usage::
class Point2D(TypedDict):
@@ -2408,39 +3018,50 @@
The type info can be accessed via the Point2D.__annotations__ dict, and
the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets.
- TypedDict supports two additional equivalent forms::
+ TypedDict supports an additional equivalent form::
- Point2D = TypedDict('Point2D', x=int, y=int, label=str)
Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str})
By default, all keys must be present in a TypedDict. It is possible
- to override this by specifying totality.
- Usage::
+ to override this by specifying totality::
- class point2D(TypedDict, total=False):
+ class Point2D(TypedDict, total=False):
x: int
y: int
- This means that a point2D TypedDict can have any of the keys omitted.A type
+ This means that a Point2D TypedDict can have any of the keys omitted. A type
checker is only expected to support a literal False or True as the value of
the total argument. True is the default, and makes all items defined in the
class body be required.
- The class syntax is only supported in Python 3.6+, while two other
- syntax forms work for Python 2.7 and 3.2+
+ The Required and NotRequired special forms can also be used to mark
+ individual keys as being required or not required::
+
+ class Point2D(TypedDict):
+ x: int # the "x" key must always be present (Required is the default)
+ y: NotRequired[int] # the "y" key can be omitted
+
+ See PEP 655 for more details on Required and NotRequired.
"""
if fields is None:
fields = kwargs
elif kwargs:
raise TypeError("TypedDict takes either a dict or keyword arguments,"
" but not both")
+ if kwargs:
+ warnings.warn(
+ "The kwargs-based syntax for TypedDict definitions is deprecated "
+ "in Python 3.11, will be removed in Python 3.13, and may not be "
+ "understood by third-party type checkers.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
ns = {'__annotations__': dict(fields)}
- try:
+ module = _caller()
+ if module is not None:
# Setting correct module is necessary to make typed dict classes pickleable.
- ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__')
- except (AttributeError, ValueError):
- pass
+ ns['__module__'] = module
return _TypedDictMeta(typename, (), ns, total=total)
@@ -2448,9 +3069,51 @@
TypedDict.__mro_entries__ = lambda bases: (_TypedDict,)
+@_SpecialForm
+def Required(self, parameters):
+ """Special typing construct to mark a TypedDict key as required.
+
+ This is mainly useful for total=False TypedDicts. For example::
+
+ class Movie(TypedDict, total=False):
+ title: Required[str]
+ year: int
+
+ m = Movie(
+ title='The Matrix', # typechecker error if key is omitted
+ year=1999,
+ )
+
+ There is no runtime checking that a required key is actually provided
+ when instantiating a related TypedDict.
+ """
+ item = _type_check(parameters, f'{self._name} accepts only a single type.')
+ return _GenericAlias(self, (item,))
+
+
+@_SpecialForm
+def NotRequired(self, parameters):
+ """Special typing construct to mark a TypedDict key as potentially missing.
+
+ For example::
+
+ class Movie(TypedDict):
+ title: str
+ year: NotRequired[int]
+
+ m = Movie(
+ title='The Matrix', # typechecker error if key is omitted
+ year=1999,
+ )
+ """
+ item = _type_check(parameters, f'{self._name} accepts only a single type.')
+ return _GenericAlias(self, (item,))
+
+
class NewType:
- """NewType creates simple unique types with almost zero
- runtime overhead. NewType(name, tp) is considered a subtype of tp
+ """NewType creates simple unique types with almost zero runtime overhead.
+
+ NewType(name, tp) is considered a subtype of tp
by static type checkers. At runtime, NewType(name, tp) returns
a dummy callable that simply returns its argument. Usage::
@@ -2467,6 +3130,8 @@
num = UserId(5) + 1 # type: int
"""
+ __call__ = _idfunc
+
def __init__(self, name, tp):
self.__qualname__ = name
if '.' in name:
@@ -2477,12 +3142,24 @@
if def_mod != 'typing':
self.__module__ = def_mod
+ def __mro_entries__(self, bases):
+ # We defined __mro_entries__ to get a better error message
+ # if a user attempts to subclass a NewType instance. bpo-46170
+ superclass_name = self.__name__
+
+ class Dummy:
+ def __init_subclass__(cls):
+ subclass_name = cls.__name__
+ raise TypeError(
+ f"Cannot subclass an instance of NewType. Perhaps you were looking for: "
+ f"`{subclass_name} = NewType({subclass_name!r}, {superclass_name})`"
+ )
+
+ return (Dummy,)
+
def __repr__(self):
return f'{self.__module__}.{self.__qualname__}'
- def __call__(self, x):
- return x
-
def __reduce__(self):
return self.__qualname__
@@ -2649,7 +3326,20 @@
pass
-class io:
+class _DeprecatedType(type):
+ def __getattribute__(cls, name):
+ if name not in ("__dict__", "__module__") and name in cls.__dict__:
+ warnings.warn(
+ f"{cls.__name__} is deprecated, import directly "
+ f"from typing instead. {cls.__name__} will be removed "
+ "in Python 3.12.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ return super().__getattribute__(name)
+
+
+class io(metaclass=_DeprecatedType):
"""Wrapper namespace for IO generic classes."""
__all__ = ['IO', 'TextIO', 'BinaryIO']
@@ -2664,7 +3354,7 @@
Pattern = _alias(stdlib_re.Pattern, 1)
Match = _alias(stdlib_re.Match, 1)
-class re:
+class re(metaclass=_DeprecatedType):
"""Wrapper namespace for re type aliases."""
__all__ = ['Pattern', 'Match']
@@ -2674,3 +3364,104 @@
re.__name__ = __name__ + '.re'
sys.modules[re.__name__] = re
+
+
+def reveal_type(obj: T, /) -> T:
+ """Reveal the inferred type of a variable.
+
+ When a static type checker encounters a call to ``reveal_type()``,
+ it will emit the inferred type of the argument::
+
+ x: int = 1
+ reveal_type(x)
+
+ Running a static type checker (e.g., mypy) on this example
+ will produce output similar to 'Revealed type is "builtins.int"'.
+
+ At runtime, the function prints the runtime type of the
+ argument and returns it unchanged.
+ """
+ print(f"Runtime type is {type(obj).__name__!r}", file=sys.stderr)
+ return obj
+
+
+def dataclass_transform(
+ *,
+ eq_default: bool = True,
+ order_default: bool = False,
+ kw_only_default: bool = False,
+ field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (),
+ **kwargs: Any,
+) -> Callable[[T], T]:
+ """Decorator to mark an object as providing dataclass-like behaviour.
+
+ The decorator can be applied to a function, class, or metaclass.
+
+ Example usage with a decorator function::
+
+ T = TypeVar("T")
+
+ @dataclass_transform()
+ def create_model(cls: type[T]) -> type[T]:
+ ...
+ return cls
+
+ @create_model
+ class CustomerModel:
+ id: int
+ name: str
+
+ On a base class::
+
+ @dataclass_transform()
+ class ModelBase: ...
+
+ class CustomerModel(ModelBase):
+ id: int
+ name: str
+
+ On a metaclass::
+
+ @dataclass_transform()
+ class ModelMeta(type): ...
+
+ class ModelBase(metaclass=ModelMeta): ...
+
+ class CustomerModel(ModelBase):
+ id: int
+ name: str
+
+ The ``CustomerModel`` classes defined above will
+ be treated by type checkers similarly to classes created with
+ ``@dataclasses.dataclass``.
+ For example, type checkers will assume these classes have
+ ``__init__`` methods that accept ``id`` and ``name``.
+
+ The arguments to this decorator can be used to customize this behavior:
+ - ``eq_default`` indicates whether the ``eq`` parameter is assumed to be
+ ``True`` or ``False`` if it is omitted by the caller.
+ - ``order_default`` indicates whether the ``order`` parameter is
+ assumed to be True or False if it is omitted by the caller.
+ - ``kw_only_default`` indicates whether the ``kw_only`` parameter is
+ assumed to be True or False if it is omitted by the caller.
+ - ``field_specifiers`` specifies a static list of supported classes
+ or functions that describe fields, similar to ``dataclasses.field()``.
+ - Arbitrary other keyword arguments are accepted in order to allow for
+ possible future extensions.
+
+ At runtime, this decorator records its arguments in the
+ ``__dataclass_transform__`` attribute on the decorated object.
+ It has no other runtime effect.
+
+ See PEP 681 for more details.
+ """
+ def decorator(cls_or_fn):
+ cls_or_fn.__dataclass_transform__ = {
+ "eq_default": eq_default,
+ "order_default": order_default,
+ "kw_only_default": kw_only_default,
+ "field_specifiers": field_specifiers,
+ "kwargs": kwargs,
+ }
+ return cls_or_fn
+ return decorator
diff --git a/common/py3-stdlib/unittest/__init__.py b/common/py3-stdlib/unittest/__init__.py
index 348dc47..005d23f 100644
--- a/common/py3-stdlib/unittest/__init__.py
+++ b/common/py3-stdlib/unittest/__init__.py
@@ -49,27 +49,30 @@
'defaultTestLoader', 'SkipTest', 'skip', 'skipIf', 'skipUnless',
'expectedFailure', 'TextTestResult', 'installHandler',
'registerResult', 'removeResult', 'removeHandler',
- 'addModuleCleanup']
+ 'addModuleCleanup', 'doModuleCleanups', 'enterModuleContext']
# Expose obsolete functions for backwards compatibility
+# bpo-5846: Deprecated in Python 3.11, scheduled for removal in Python 3.13.
__all__.extend(['getTestCaseNames', 'makeSuite', 'findTestCases'])
__unittest = True
from .result import TestResult
from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip,
- skipIf, skipUnless, expectedFailure)
+ skipIf, skipUnless, expectedFailure, doModuleCleanups,
+ enterModuleContext)
from .suite import BaseTestSuite, TestSuite
-from .loader import (TestLoader, defaultTestLoader, makeSuite, getTestCaseNames,
- findTestCases)
+from .loader import TestLoader, defaultTestLoader
from .main import TestProgram, main
from .runner import TextTestRunner, TextTestResult
from .signals import installHandler, registerResult, removeResult, removeHandler
# IsolatedAsyncioTestCase will be imported lazily.
+from .loader import makeSuite, getTestCaseNames, findTestCases
# deprecated
_TextTestResult = TextTestResult
+
# There are no tests here, so don't try to run anything discovered from
# introspecting the symbols (e.g. FunctionTestCase). Instead, all our
# tests come from within unittest.test.
diff --git a/common/py3-stdlib/unittest/async_case.py b/common/py3-stdlib/unittest/async_case.py
index d9c694e..bd2a471 100644
--- a/common/py3-stdlib/unittest/async_case.py
+++ b/common/py3-stdlib/unittest/async_case.py
@@ -1,5 +1,7 @@
import asyncio
+import contextvars
import inspect
+import warnings
from .case import TestCase
@@ -32,8 +34,8 @@
def __init__(self, methodName='runTest'):
super().__init__(methodName)
- self._asyncioTestLoop = None
- self._asyncioCallsQueue = None
+ self._asyncioRunner = None
+ self._asyncioTestContext = contextvars.copy_context()
async def asyncSetUp(self):
pass
@@ -56,115 +58,85 @@
# 3. Regular "def func()" that returns awaitable object
self.addCleanup(*(func, *args), **kwargs)
+ async def enterAsyncContext(self, cm):
+ """Enters the supplied asynchronous context manager.
+
+ If successful, also adds its __aexit__ method as a cleanup
+ function and returns the result of the __aenter__ method.
+ """
+ # We look up the special methods on the type to match the with
+ # statement.
+ cls = type(cm)
+ try:
+ enter = cls.__aenter__
+ exit = cls.__aexit__
+ except AttributeError:
+ raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
+ f"not support the asynchronous context manager protocol"
+ ) from None
+ result = await enter(cm)
+ self.addAsyncCleanup(exit, cm, None, None, None)
+ return result
+
def _callSetUp(self):
- self.setUp()
+ # Force loop to be initialized and set as the current loop
+ # so that setUp functions can use get_event_loop() and get the
+ # correct loop instance.
+ self._asyncioRunner.get_loop()
+ self._asyncioTestContext.run(self.setUp)
self._callAsync(self.asyncSetUp)
def _callTestMethod(self, method):
- self._callMaybeAsync(method)
+ if self._callMaybeAsync(method) is not None:
+ warnings.warn(f'It is deprecated to return a value that is not None from a '
+ f'test case ({method})', DeprecationWarning, stacklevel=4)
def _callTearDown(self):
self._callAsync(self.asyncTearDown)
- self.tearDown()
+ self._asyncioTestContext.run(self.tearDown)
def _callCleanup(self, function, *args, **kwargs):
self._callMaybeAsync(function, *args, **kwargs)
def _callAsync(self, func, /, *args, **kwargs):
- assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized'
- ret = func(*args, **kwargs)
- assert inspect.isawaitable(ret), f'{func!r} returned non-awaitable'
- fut = self._asyncioTestLoop.create_future()
- self._asyncioCallsQueue.put_nowait((fut, ret))
- return self._asyncioTestLoop.run_until_complete(fut)
+ assert self._asyncioRunner is not None, 'asyncio runner is not initialized'
+ assert inspect.iscoroutinefunction(func), f'{func!r} is not an async function'
+ return self._asyncioRunner.run(
+ func(*args, **kwargs),
+ context=self._asyncioTestContext
+ )
def _callMaybeAsync(self, func, /, *args, **kwargs):
- assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized'
- ret = func(*args, **kwargs)
- if inspect.isawaitable(ret):
- fut = self._asyncioTestLoop.create_future()
- self._asyncioCallsQueue.put_nowait((fut, ret))
- return self._asyncioTestLoop.run_until_complete(fut)
+ assert self._asyncioRunner is not None, 'asyncio runner is not initialized'
+ if inspect.iscoroutinefunction(func):
+ return self._asyncioRunner.run(
+ func(*args, **kwargs),
+ context=self._asyncioTestContext,
+ )
else:
- return ret
+ return self._asyncioTestContext.run(func, *args, **kwargs)
- async def _asyncioLoopRunner(self, fut):
- self._asyncioCallsQueue = queue = asyncio.Queue()
- fut.set_result(None)
- while True:
- query = await queue.get()
- queue.task_done()
- if query is None:
- return
- fut, awaitable = query
- try:
- ret = await awaitable
- if not fut.cancelled():
- fut.set_result(ret)
- except (SystemExit, KeyboardInterrupt):
- raise
- except (BaseException, asyncio.CancelledError) as ex:
- if not fut.cancelled():
- fut.set_exception(ex)
+ def _setupAsyncioRunner(self):
+ assert self._asyncioRunner is None, 'asyncio runner is already initialized'
+ runner = asyncio.Runner(debug=True)
+ self._asyncioRunner = runner
- def _setupAsyncioLoop(self):
- assert self._asyncioTestLoop is None, 'asyncio test loop already initialized'
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- loop.set_debug(True)
- self._asyncioTestLoop = loop
- fut = loop.create_future()
- self._asyncioCallsTask = loop.create_task(self._asyncioLoopRunner(fut))
- loop.run_until_complete(fut)
-
- def _tearDownAsyncioLoop(self):
- assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized'
- loop = self._asyncioTestLoop
- self._asyncioTestLoop = None
- self._asyncioCallsQueue.put_nowait(None)
- loop.run_until_complete(self._asyncioCallsQueue.join())
-
- try:
- # cancel all tasks
- to_cancel = asyncio.all_tasks(loop)
- if not to_cancel:
- return
-
- for task in to_cancel:
- task.cancel()
-
- loop.run_until_complete(
- asyncio.gather(*to_cancel, return_exceptions=True))
-
- for task in to_cancel:
- if task.cancelled():
- continue
- if task.exception() is not None:
- loop.call_exception_handler({
- 'message': 'unhandled exception during test shutdown',
- 'exception': task.exception(),
- 'task': task,
- })
- # shutdown asyncgens
- loop.run_until_complete(loop.shutdown_asyncgens())
- finally:
- # Prevent our executor environment from leaking to future tests.
- loop.run_until_complete(loop.shutdown_default_executor())
- asyncio.set_event_loop(None)
- loop.close()
+ def _tearDownAsyncioRunner(self):
+ runner = self._asyncioRunner
+ runner.close()
def run(self, result=None):
- self._setupAsyncioLoop()
+ self._setupAsyncioRunner()
try:
return super().run(result)
finally:
- self._tearDownAsyncioLoop()
+ self._tearDownAsyncioRunner()
def debug(self):
- self._setupAsyncioLoop()
+ self._setupAsyncioRunner()
super().debug()
- self._tearDownAsyncioLoop()
+ self._tearDownAsyncioRunner()
def __del__(self):
- if self._asyncioTestLoop is not None:
- self._tearDownAsyncioLoop()
+ if self._asyncioRunner is not None:
+ self._tearDownAsyncioRunner()
diff --git a/common/py3-stdlib/unittest/case.py b/common/py3-stdlib/unittest/case.py
index 50100e9..c4aa2d7 100644
--- a/common/py3-stdlib/unittest/case.py
+++ b/common/py3-stdlib/unittest/case.py
@@ -47,12 +47,10 @@
self.result = result
self.result_supports_subtests = hasattr(result, "addSubTest")
self.success = True
- self.skipped = []
self.expectedFailure = None
- self.errors = []
@contextlib.contextmanager
- def testPartExecutor(self, test_case, isTest=False):
+ def testPartExecutor(self, test_case, subTest=False):
old_success = self.success
self.success = True
try:
@@ -61,7 +59,7 @@
raise
except SkipTest as e:
self.success = False
- self.skipped.append((test_case, str(e)))
+ _addSkip(self.result, test_case, str(e))
except _ShouldStop:
pass
except:
@@ -70,27 +68,65 @@
self.expectedFailure = exc_info
else:
self.success = False
- self.errors.append((test_case, exc_info))
+ if subTest:
+ self.result.addSubTest(test_case.test_case, test_case, exc_info)
+ else:
+ _addError(self.result, test_case, exc_info)
# explicitly break a reference cycle:
# exc_info -> frame -> exc_info
exc_info = None
else:
- if self.result_supports_subtests and self.success:
- self.errors.append((test_case, None))
+ if subTest and self.success:
+ self.result.addSubTest(test_case.test_case, test_case, None)
finally:
self.success = self.success and old_success
+def _addSkip(result, test_case, reason):
+ addSkip = getattr(result, 'addSkip', None)
+ if addSkip is not None:
+ addSkip(test_case, reason)
+ else:
+ warnings.warn("TestResult has no addSkip method, skips not reported",
+ RuntimeWarning, 2)
+ result.addSuccess(test_case)
+
+def _addError(result, test, exc_info):
+ if result is not None and exc_info is not None:
+ if issubclass(exc_info[0], test.failureException):
+ result.addFailure(test, exc_info)
+ else:
+ result.addError(test, exc_info)
+
def _id(obj):
return obj
+def _enter_context(cm, addcleanup):
+ # We look up the special methods on the type to match the with
+ # statement.
+ cls = type(cm)
+ try:
+ enter = cls.__enter__
+ exit = cls.__exit__
+ except AttributeError:
+ raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
+ f"not support the context manager protocol") from None
+ result = enter(cm)
+ addcleanup(exit, cm, None, None, None)
+ return result
+
+
_module_cleanups = []
def addModuleCleanup(function, /, *args, **kwargs):
"""Same as addCleanup, except the cleanup items are called even if
setUpModule fails (unlike tearDownModule)."""
_module_cleanups.append((function, args, kwargs))
+def enterModuleContext(cm):
+ """Same as enterContext, but module-wide."""
+ return _enter_context(cm, addModuleCleanup)
+
def doModuleCleanups():
"""Execute all module cleanup functions. Normally called for you after
@@ -409,12 +445,25 @@
Cleanup items are called even if setUp fails (unlike tearDown)."""
self._cleanups.append((function, args, kwargs))
+ def enterContext(self, cm):
+ """Enters the supplied context manager.
+
+ If successful, also adds its __exit__ method as a cleanup
+ function and returns the result of the __enter__ method.
+ """
+ return _enter_context(cm, self.addCleanup)
+
@classmethod
def addClassCleanup(cls, function, /, *args, **kwargs):
"""Same as addCleanup, except the cleanup items are called even if
setUpClass fails (unlike tearDownClass)."""
cls._class_cleanups.append((function, args, kwargs))
+ @classmethod
+ def enterClassContext(cls, cm):
+ """Same as enterContext, but class-wide."""
+ return _enter_context(cm, cls.addClassCleanup)
+
def setUp(self):
"Hook method for setting up the test fixture before exercising it."
pass
@@ -461,21 +510,12 @@
return hash((type(self), self._testMethodName))
def __str__(self):
- return "%s (%s)" % (self._testMethodName, strclass(self.__class__))
+ return "%s (%s.%s)" % (self._testMethodName, strclass(self.__class__), self._testMethodName)
def __repr__(self):
return "<%s testMethod=%s>" % \
(strclass(self.__class__), self._testMethodName)
- def _addSkip(self, result, test_case, reason):
- addSkip = getattr(result, 'addSkip', None)
- if addSkip is not None:
- addSkip(test_case, reason)
- else:
- warnings.warn("TestResult has no addSkip method, skips not reported",
- RuntimeWarning, 2)
- result.addSuccess(test_case)
-
@contextlib.contextmanager
def subTest(self, msg=_subtest_msg_sentinel, **params):
"""Return a context manager that will return the enclosed block
@@ -494,7 +534,7 @@
params_map = parent.params.new_child(params)
self._subtest = _SubTest(self, msg, params_map)
try:
- with self._outcome.testPartExecutor(self._subtest, isTest=True):
+ with self._outcome.testPartExecutor(self._subtest, subTest=True):
yield
if not self._outcome.success:
result = self._outcome.result
@@ -507,16 +547,6 @@
finally:
self._subtest = parent
- def _feedErrorsToResult(self, result, errors):
- for test, exc_info in errors:
- if isinstance(test, _SubTest):
- result.addSubTest(test.test_case, test, exc_info)
- elif exc_info is not None:
- if issubclass(exc_info[0], self.failureException):
- result.addFailure(test, exc_info)
- else:
- result.addError(test, exc_info)
-
def _addExpectedFailure(self, result, exc_info):
try:
addExpectedFailure = result.addExpectedFailure
@@ -546,7 +576,9 @@
self.setUp()
def _callTestMethod(self, method):
- method()
+ if method() is not None:
+ warnings.warn(f'It is deprecated to return a value that is not None from a '
+ f'test case ({method})', DeprecationWarning, stacklevel=3)
def _callTearDown(self):
self.tearDown()
@@ -572,7 +604,7 @@
# If the class or method was skipped.
skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
or getattr(testMethod, '__unittest_skip_why__', ''))
- self._addSkip(result, self, skip_why)
+ _addSkip(result, self, skip_why)
return result
expecting_failure = (
@@ -587,16 +619,13 @@
self._callSetUp()
if outcome.success:
outcome.expecting_failure = expecting_failure
- with outcome.testPartExecutor(self, isTest=True):
+ with outcome.testPartExecutor(self):
self._callTestMethod(testMethod)
outcome.expecting_failure = False
with outcome.testPartExecutor(self):
self._callTearDown()
-
self.doCleanups()
- for test, reason in outcome.skipped:
- self._addSkip(result, test, reason)
- self._feedErrorsToResult(result, outcome.errors)
+
if outcome.success:
if expecting_failure:
if outcome.expectedFailure:
@@ -607,11 +636,10 @@
result.addSuccess(self)
return result
finally:
- # explicitly break reference cycles:
- # outcome.errors -> frame -> outcome -> outcome.errors
+ # explicitly break reference cycle:
# outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure
- outcome.errors.clear()
outcome.expectedFailure = None
+ outcome = None
# clear the outcome, no more needed
self._outcome = None
@@ -1146,8 +1174,7 @@
def assertDictContainsSubset(self, subset, dictionary, msg=None):
"""Checks whether dictionary is a superset of subset."""
warnings.warn('assertDictContainsSubset is deprecated',
- DeprecationWarning,
- stacklevel=2)
+ DeprecationWarning)
missing = []
mismatched = []
for key, value in subset.items():
diff --git a/common/py3-stdlib/unittest/loader.py b/common/py3-stdlib/unittest/loader.py
index ba7105e..7e6ce2f 100644
--- a/common/py3-stdlib/unittest/loader.py
+++ b/common/py3-stdlib/unittest/loader.py
@@ -286,8 +286,6 @@
self._top_level_dir = top_level_dir
is_not_importable = False
- is_namespace = False
- tests = []
if os.path.isdir(os.path.abspath(start_dir)):
start_dir = os.path.abspath(start_dir)
if start_dir != top_level_dir:
@@ -303,50 +301,25 @@
top_part = start_dir.split('.')[0]
try:
start_dir = os.path.abspath(
- os.path.dirname((the_module.__file__)))
+ os.path.dirname((the_module.__file__)))
except AttributeError:
- # look for namespace packages
- try:
- spec = the_module.__spec__
- except AttributeError:
- spec = None
-
- if spec and spec.loader is None:
- if spec.submodule_search_locations is not None:
- is_namespace = True
-
- for path in the_module.__path__:
- if (not set_implicit_top and
- not path.startswith(top_level_dir)):
- continue
- self._top_level_dir = \
- (path.split(the_module.__name__
- .replace(".", os.path.sep))[0])
- tests.extend(self._find_tests(path,
- pattern,
- namespace=True))
- elif the_module.__name__ in sys.builtin_module_names:
+ if the_module.__name__ in sys.builtin_module_names:
# builtin module
raise TypeError('Can not use builtin modules '
'as dotted module names') from None
else:
raise TypeError(
- 'don\'t know how to discover from {!r}'
- .format(the_module)) from None
+ f"don't know how to discover from {the_module!r}"
+ ) from None
if set_implicit_top:
- if not is_namespace:
- self._top_level_dir = \
- self._get_directory_containing_module(top_part)
- sys.path.remove(top_level_dir)
- else:
- sys.path.remove(top_level_dir)
+ self._top_level_dir = self._get_directory_containing_module(top_part)
+ sys.path.remove(top_level_dir)
if is_not_importable:
raise ImportError('Start directory is not importable: %r' % start_dir)
- if not is_namespace:
- tests = list(self._find_tests(start_dir, pattern))
+ tests = list(self._find_tests(start_dir, pattern))
return self.suiteClass(tests)
def _get_directory_containing_module(self, module_name):
@@ -381,7 +354,7 @@
# override this method to use alternative matching strategy
return fnmatch(path, pattern)
- def _find_tests(self, start_dir, pattern, namespace=False):
+ def _find_tests(self, start_dir, pattern):
"""Used by discovery. Yields test suites it loads."""
# Handle the __init__ in this package
name = self._get_name_from_path(start_dir)
@@ -390,8 +363,7 @@
if name != '.' and name not in self._loading_packages:
# name is in self._loading_packages while we have called into
# loadTestsFromModule with name.
- tests, should_recurse = self._find_test_path(
- start_dir, pattern, namespace)
+ tests, should_recurse = self._find_test_path(start_dir, pattern)
if tests is not None:
yield tests
if not should_recurse:
@@ -402,8 +374,7 @@
paths = sorted(os.listdir(start_dir))
for path in paths:
full_path = os.path.join(start_dir, path)
- tests, should_recurse = self._find_test_path(
- full_path, pattern, namespace)
+ tests, should_recurse = self._find_test_path(full_path, pattern)
if tests is not None:
yield tests
if should_recurse:
@@ -411,11 +382,11 @@
name = self._get_name_from_path(full_path)
self._loading_packages.add(name)
try:
- yield from self._find_tests(full_path, pattern, namespace)
+ yield from self._find_tests(full_path, pattern)
finally:
self._loading_packages.discard(name)
- def _find_test_path(self, full_path, pattern, namespace=False):
+ def _find_test_path(self, full_path, pattern):
"""Used by discovery.
Loads tests from a single file, or a directories' __init__.py when
@@ -459,8 +430,7 @@
msg % (mod_name, module_dir, expected_dir))
return self.loadTestsFromModule(module, pattern=pattern), False
elif os.path.isdir(full_path):
- if (not namespace and
- not os.path.isfile(os.path.join(full_path, '__init__.py'))):
+ if not os.path.isfile(os.path.join(full_path, '__init__.py')):
return None, False
load_tests = None
@@ -494,6 +464,9 @@
defaultTestLoader = TestLoader()
+# These functions are considered obsolete for long time.
+# They will be removed in Python 3.13.
+
def _makeLoader(prefix, sortUsing, suiteClass=None, testNamePatterns=None):
loader = TestLoader()
loader.sortTestMethodsUsing = sortUsing
@@ -504,14 +477,32 @@
return loader
def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp, testNamePatterns=None):
+ import warnings
+ warnings.warn(
+ "unittest.getTestCaseNames() is deprecated and will be removed in Python 3.13. "
+ "Please use unittest.TestLoader.getTestCaseNames() instead.",
+ DeprecationWarning, stacklevel=2
+ )
return _makeLoader(prefix, sortUsing, testNamePatterns=testNamePatterns).getTestCaseNames(testCaseClass)
def makeSuite(testCaseClass, prefix='test', sortUsing=util.three_way_cmp,
suiteClass=suite.TestSuite):
+ import warnings
+ warnings.warn(
+ "unittest.makeSuite() is deprecated and will be removed in Python 3.13. "
+ "Please use unittest.TestLoader.loadTestsFromTestCase() instead.",
+ DeprecationWarning, stacklevel=2
+ )
return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromTestCase(
testCaseClass)
def findTestCases(module, prefix='test', sortUsing=util.three_way_cmp,
suiteClass=suite.TestSuite):
+ import warnings
+ warnings.warn(
+ "unittest.findTestCases() is deprecated and will be removed in Python 3.13. "
+ "Please use unittest.TestLoader.loadTestsFromModule() instead.",
+ DeprecationWarning, stacklevel=2
+ )
return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromModule(\
module)
diff --git a/common/py3-stdlib/unittest/main.py b/common/py3-stdlib/unittest/main.py
index 88a188c..046fbd3 100644
--- a/common/py3-stdlib/unittest/main.py
+++ b/common/py3-stdlib/unittest/main.py
@@ -3,6 +3,7 @@
import sys
import argparse
import os
+import warnings
from . import loader, runner
from .signals import installHandler
@@ -101,6 +102,8 @@
self.runTests()
def usageExit(self, msg=None):
+ warnings.warn("TestProgram.usageExit() is deprecated and will be"
+ " removed in Python 3.13", DeprecationWarning)
if msg:
print(msg)
if self._discovery_parser is None:
diff --git a/common/py3-stdlib/unittest/mock.py b/common/py3-stdlib/unittest/mock.py
index 7453dfa..e78eb35 100644
--- a/common/py3-stdlib/unittest/mock.py
+++ b/common/py3-stdlib/unittest/mock.py
@@ -30,6 +30,7 @@
import pprint
import sys
import builtins
+import pkgutil
from asyncio import iscoroutinefunction
from types import CodeType, ModuleType, MethodType
from unittest.util import safe_repr
@@ -97,6 +98,12 @@
func = func.__init__
# Skip the `self` argument in __init__
eat_self = True
+ elif isinstance(func, (classmethod, staticmethod)):
+ if isinstance(func, classmethod):
+ # Skip the `cls` argument of a class method
+ eat_self = True
+ # Use the original decorated method to extract the correct function signature
+ func = func.__func__
elif not isinstance(func, FunctionTypes):
# If we really want to model an instance of the passed type,
# __call__ should be looked up, not __init__.
@@ -497,6 +504,9 @@
def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False,
_eat_self=False):
+ if _is_instance_mock(spec):
+ raise InvalidSpecError(f'Cannot spec a Mock object. [object={spec!r}]')
+
_spec_class = None
_spec_signature = None
_spec_asyncs = []
@@ -1224,6 +1234,11 @@
this is a new Mock (created on first access). See the
`return_value` attribute.
+ * `unsafe`: By default, accessing any attribute whose name starts with
+ *assert*, *assret*, *asert*, *aseert* or *assrt* will raise an
+ AttributeError. Passing `unsafe=True` will allow access to
+ these attributes.
+
* `wraps`: Item for the mock object to wrap. If `wraps` is not None then
calling the Mock will pass the call through to the wrapped object
(returning the real result). Attribute access on the mock will return a
@@ -1243,25 +1258,6 @@
"""
-def _dot_lookup(thing, comp, import_path):
- try:
- return getattr(thing, comp)
- except AttributeError:
- __import__(import_path)
- return getattr(thing, comp)
-
-
-def _importer(target):
- components = target.split('.')
- import_path = components.pop(0)
- thing = __import__(import_path)
-
- for comp in components:
- import_path += ".%s" % comp
- thing = _dot_lookup(thing, comp, import_path)
- return thing
-
-
# _check_spec_arg_typos takes kwargs from commands like patch and checks that
# they don't contain common misspellings of arguments related to autospeccing.
def _check_spec_arg_typos(kwargs_to_check):
@@ -1615,8 +1611,7 @@
except (TypeError, ValueError, AttributeError):
raise TypeError(
f"Need a valid target to patch. You supplied: {target!r}")
- getter = lambda: _importer(target)
- return getter, attribute
+ return partial(pkgutil.resolve_name, target), attribute
def _patch_object(
@@ -1671,7 +1666,7 @@
for choosing which methods to wrap.
"""
if type(target) is str:
- getter = lambda: _importer(target)
+ getter = partial(pkgutil.resolve_name, target)
else:
getter = lambda: target
@@ -1869,7 +1864,7 @@
def _patch_dict(self):
values = self.values
if isinstance(self.in_dict, str):
- self.in_dict = _importer(self.in_dict)
+ self.in_dict = pkgutil.resolve_name(self.in_dict)
in_dict = self.in_dict
clear = self.clear
@@ -1970,7 +1965,7 @@
)
numerics = (
- "add sub mul matmul div floordiv mod lshift rshift and xor or pow truediv"
+ "add sub mul matmul truediv floordiv mod lshift rshift and xor or pow"
)
inplace = ' '.join('i%s' % n for n in numerics.split())
right = ' '.join('r%s' % n for n in numerics.split())
@@ -1982,7 +1977,7 @@
_non_defaults = {
'__get__', '__set__', '__delete__', '__reversed__', '__missing__',
'__reduce__', '__reduce_ex__', '__getinitargs__', '__getnewargs__',
- '__getstate__', '__setstate__', '__getformat__', '__setformat__',
+ '__getstate__', '__setstate__', '__getformat__',
'__repr__', '__dir__', '__subclasses__', '__format__',
'__getnewargs_ex__',
}
@@ -2212,7 +2207,15 @@
self.__dict__['_mock_await_args'] = None
self.__dict__['_mock_await_args_list'] = _CallList()
code_mock = NonCallableMock(spec_set=CodeType)
- code_mock.co_flags = inspect.CO_COROUTINE
+ code_mock.co_flags = (
+ inspect.CO_COROUTINE
+ + inspect.CO_VARARGS
+ + inspect.CO_VARKEYWORDS
+ )
+ code_mock.co_argcount = 0
+ code_mock.co_varnames = ('args', 'kwargs')
+ code_mock.co_posonlyargcount = 0
+ code_mock.co_kwonlyargcount = 0
self.__dict__['__code__'] = code_mock
self.__dict__['__name__'] = 'AsyncMock'
self.__dict__['__defaults__'] = tuple()
@@ -2236,7 +2239,7 @@
try:
result = next(effect)
except StopIteration:
- # It is impossible to propogate a StopIteration
+ # It is impossible to propagate a StopIteration
# through coroutines because of PEP 479
raise StopAsyncIteration
if _is_exception(result):
@@ -2835,6 +2838,7 @@
file_spec = None
+open_spec = None
def _to_stream(read_data):
@@ -2891,8 +2895,12 @@
import _io
file_spec = list(set(dir(_io.TextIOWrapper)).union(set(dir(_io.BytesIO))))
+ global open_spec
+ if open_spec is None:
+ import _io
+ open_spec = list(set(dir(_io.open)))
if mock is None:
- mock = MagicMock(name='open', spec=open)
+ mock = MagicMock(name='open', spec=open_spec)
handle = MagicMock(spec=file_spec)
handle.__enter__.return_value = handle
diff --git a/common/py3-stdlib/unittest/runner.py b/common/py3-stdlib/unittest/runner.py
index caf1590..cb452c7 100644
--- a/common/py3-stdlib/unittest/runner.py
+++ b/common/py3-stdlib/unittest/runner.py
@@ -5,6 +5,7 @@
import warnings
from . import result
+from .case import _SubTest
from .signals import registerResult
__unittest = True
@@ -40,6 +41,7 @@
self.showAll = verbosity > 1
self.dots = verbosity == 1
self.descriptions = descriptions
+ self._newline = True
def getDescription(self, test):
doc_first_line = test.shortDescription()
@@ -54,12 +56,40 @@
self.stream.write(self.getDescription(test))
self.stream.write(" ... ")
self.stream.flush()
+ self._newline = False
+
+ def _write_status(self, test, status):
+ is_subtest = isinstance(test, _SubTest)
+ if is_subtest or self._newline:
+ if not self._newline:
+ self.stream.writeln()
+ if is_subtest:
+ self.stream.write(" ")
+ self.stream.write(self.getDescription(test))
+ self.stream.write(" ... ")
+ self.stream.writeln(status)
+ self.stream.flush()
+ self._newline = True
+
+ def addSubTest(self, test, subtest, err):
+ if err is not None:
+ if self.showAll:
+ if issubclass(err[0], subtest.failureException):
+ self._write_status(subtest, "FAIL")
+ else:
+ self._write_status(subtest, "ERROR")
+ elif self.dots:
+ if issubclass(err[0], subtest.failureException):
+ self.stream.write('F')
+ else:
+ self.stream.write('E')
+ self.stream.flush()
+ super(TextTestResult, self).addSubTest(test, subtest, err)
def addSuccess(self, test):
super(TextTestResult, self).addSuccess(test)
if self.showAll:
- self.stream.writeln("ok")
- self.stream.flush()
+ self._write_status(test, "ok")
elif self.dots:
self.stream.write('.')
self.stream.flush()
@@ -67,8 +97,7 @@
def addError(self, test, err):
super(TextTestResult, self).addError(test, err)
if self.showAll:
- self.stream.writeln("ERROR")
- self.stream.flush()
+ self._write_status(test, "ERROR")
elif self.dots:
self.stream.write('E')
self.stream.flush()
@@ -76,8 +105,7 @@
def addFailure(self, test, err):
super(TextTestResult, self).addFailure(test, err)
if self.showAll:
- self.stream.writeln("FAIL")
- self.stream.flush()
+ self._write_status(test, "FAIL")
elif self.dots:
self.stream.write('F')
self.stream.flush()
@@ -85,8 +113,7 @@
def addSkip(self, test, reason):
super(TextTestResult, self).addSkip(test, reason)
if self.showAll:
- self.stream.writeln("skipped {0!r}".format(reason))
- self.stream.flush()
+ self._write_status(test, "skipped {0!r}".format(reason))
elif self.dots:
self.stream.write("s")
self.stream.flush()
@@ -115,6 +142,12 @@
self.stream.flush()
self.printErrorList('ERROR', self.errors)
self.printErrorList('FAIL', self.failures)
+ unexpectedSuccesses = getattr(self, 'unexpectedSuccesses', ())
+ if unexpectedSuccesses:
+ self.stream.writeln(self.separator1)
+ for test in unexpectedSuccesses:
+ self.stream.writeln(f"UNEXPECTED SUCCESS: {self.getDescription(test)}")
+ self.stream.flush()
def printErrorList(self, flavour, errors):
for test, err in errors:
diff --git a/common/py3-stdlib/unittest/test/__init__.py b/common/py3-stdlib/unittest/test/__init__.py
index cdae8a7..143f4ab 100644
--- a/common/py3-stdlib/unittest/test/__init__.py
+++ b/common/py3-stdlib/unittest/test/__init__.py
@@ -11,7 +11,10 @@
for fn in os.listdir(here):
if fn.startswith("test") and fn.endswith(".py"):
modname = "unittest.test." + fn[:-3]
- __import__(modname)
+ try:
+ __import__(modname)
+ except unittest.SkipTest:
+ continue
module = sys.modules[modname]
suite.addTest(loader.loadTestsFromModule(module))
suite.addTest(loader.loadTestsFromName('unittest.test.testmock'))
diff --git a/common/py3-stdlib/unittest/test/test_async_case.py b/common/py3-stdlib/unittest/test/test_async_case.py
index a68ae86..a465103 100644
--- a/common/py3-stdlib/unittest/test/test_async_case.py
+++ b/common/py3-stdlib/unittest/test/test_async_case.py
@@ -1,7 +1,10 @@
import asyncio
+import contextvars
import unittest
from test import support
+support.requires_working_socket(module=True)
+
class MyException(Exception):
pass
@@ -11,6 +14,32 @@
asyncio.set_event_loop_policy(None)
+class TestCM:
+ def __init__(self, ordering, enter_result=None):
+ self.ordering = ordering
+ self.enter_result = enter_result
+
+ async def __aenter__(self):
+ self.ordering.append('enter')
+ return self.enter_result
+
+ async def __aexit__(self, *exc_info):
+ self.ordering.append('exit')
+
+
+class LacksEnterAndExit:
+ pass
+class LacksEnter:
+ async def __aexit__(self, *exc_info):
+ pass
+class LacksExit:
+ async def __aenter__(self):
+ pass
+
+
+VAR = contextvars.ContextVar('VAR', default=())
+
+
class TestAsyncCase(unittest.TestCase):
maxDiff = None
@@ -35,66 +64,84 @@
def setUp(self):
self.assertEqual(events, [])
events.append('setUp')
+ VAR.set(VAR.get() + ('setUp',))
self.addCleanup(self.on_cleanup1)
self.addAsyncCleanup(self.on_cleanup2)
async def asyncSetUp(self):
self.assertEqual(events, expected[:1])
events.append('asyncSetUp')
+ VAR.set(VAR.get() + ('asyncSetUp',))
self.addCleanup(self.on_cleanup3)
self.addAsyncCleanup(self.on_cleanup4)
async def test_func(self):
self.assertEqual(events, expected[:2])
events.append('test')
+ VAR.set(VAR.get() + ('test',))
self.addCleanup(self.on_cleanup5)
self.addAsyncCleanup(self.on_cleanup6)
async def asyncTearDown(self):
self.assertEqual(events, expected[:3])
+ VAR.set(VAR.get() + ('asyncTearDown',))
events.append('asyncTearDown')
def tearDown(self):
self.assertEqual(events, expected[:4])
events.append('tearDown')
+ VAR.set(VAR.get() + ('tearDown',))
def on_cleanup1(self):
self.assertEqual(events, expected[:10])
events.append('cleanup1')
+ VAR.set(VAR.get() + ('cleanup1',))
+ nonlocal cvar
+ cvar = VAR.get()
async def on_cleanup2(self):
self.assertEqual(events, expected[:9])
events.append('cleanup2')
+ VAR.set(VAR.get() + ('cleanup2',))
def on_cleanup3(self):
self.assertEqual(events, expected[:8])
events.append('cleanup3')
+ VAR.set(VAR.get() + ('cleanup3',))
async def on_cleanup4(self):
self.assertEqual(events, expected[:7])
events.append('cleanup4')
+ VAR.set(VAR.get() + ('cleanup4',))
def on_cleanup5(self):
self.assertEqual(events, expected[:6])
events.append('cleanup5')
+ VAR.set(VAR.get() + ('cleanup5',))
async def on_cleanup6(self):
self.assertEqual(events, expected[:5])
events.append('cleanup6')
+ VAR.set(VAR.get() + ('cleanup6',))
events = []
+ cvar = ()
test = Test("test_func")
result = test.run()
self.assertEqual(result.errors, [])
self.assertEqual(result.failures, [])
self.assertEqual(events, expected)
+ self.assertEqual(cvar, tuple(expected))
events = []
+ cvar = ()
test = Test("test_func")
test.debug()
self.assertEqual(events, expected)
+ self.assertEqual(cvar, tuple(expected))
test.doCleanups()
self.assertEqual(events, expected)
+ self.assertEqual(cvar, tuple(expected))
def test_exception_in_setup(self):
class Test(unittest.IsolatedAsyncioTestCase):
@@ -122,7 +169,7 @@
events = []
test = Test("test_func")
- self.addCleanup(test._tearDownAsyncioLoop)
+ self.addCleanup(test._tearDownAsyncioRunner)
try:
test.debug()
except MyException:
@@ -158,7 +205,7 @@
events = []
test = Test("test_func")
- self.addCleanup(test._tearDownAsyncioLoop)
+ self.addCleanup(test._tearDownAsyncioRunner)
try:
test.debug()
except MyException:
@@ -194,7 +241,7 @@
events = []
test = Test("test_func")
- self.addCleanup(test._tearDownAsyncioLoop)
+ self.addCleanup(test._tearDownAsyncioRunner)
try:
test.debug()
except MyException:
@@ -236,7 +283,7 @@
events = []
test = Test("test_func")
- self.addCleanup(test._tearDownAsyncioLoop)
+ self.addCleanup(test._tearDownAsyncioRunner)
try:
test.debug()
except MyException:
@@ -247,6 +294,37 @@
test.doCleanups()
self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2', 'cleanup1'])
+ def test_deprecation_of_return_val_from_test(self):
+ # Issue 41322 - deprecate return of value that is not None from a test
+ class Nothing:
+ def __eq__(self, o):
+ return o is None
+ class Test(unittest.IsolatedAsyncioTestCase):
+ async def test1(self):
+ return 1
+ async def test2(self):
+ yield 1
+ async def test3(self):
+ return Nothing()
+
+ with self.assertWarns(DeprecationWarning) as w:
+ Test('test1').run()
+ self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
+ self.assertIn('test1', str(w.warning))
+ self.assertEqual(w.filename, __file__)
+
+ with self.assertWarns(DeprecationWarning) as w:
+ Test('test2').run()
+ self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
+ self.assertIn('test2', str(w.warning))
+ self.assertEqual(w.filename, __file__)
+
+ with self.assertWarns(DeprecationWarning) as w:
+ Test('test3').run()
+ self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
+ self.assertIn('test3', str(w.warning))
+ self.assertEqual(w.filename, __file__)
+
def test_cleanups_interleave_order(self):
events = []
@@ -315,6 +393,36 @@
output = test.run()
self.assertTrue(cancelled)
+ def test_enterAsyncContext(self):
+ events = []
+
+ class Test(unittest.IsolatedAsyncioTestCase):
+ async def test_func(slf):
+ slf.addAsyncCleanup(events.append, 'cleanup1')
+ cm = TestCM(events, 42)
+ self.assertEqual(await slf.enterAsyncContext(cm), 42)
+ slf.addAsyncCleanup(events.append, 'cleanup2')
+ events.append('test')
+
+ test = Test('test_func')
+ output = test.run()
+ self.assertTrue(output.wasSuccessful(), output)
+ self.assertEqual(events, ['enter', 'test', 'cleanup2', 'exit', 'cleanup1'])
+
+ def test_enterAsyncContext_arg_errors(self):
+ class Test(unittest.IsolatedAsyncioTestCase):
+ async def test_func(slf):
+ with self.assertRaisesRegex(TypeError, 'asynchronous context manager'):
+ await slf.enterAsyncContext(LacksEnterAndExit())
+ with self.assertRaisesRegex(TypeError, 'asynchronous context manager'):
+ await slf.enterAsyncContext(LacksEnter())
+ with self.assertRaisesRegex(TypeError, 'asynchronous context manager'):
+ await slf.enterAsyncContext(LacksExit())
+
+ test = Test('test_func')
+ output = test.run()
+ self.assertTrue(output.wasSuccessful())
+
def test_debug_cleanup_same_loop(self):
class Test(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
@@ -349,7 +457,7 @@
events = []
test = Test("test_func")
- self.addCleanup(test._tearDownAsyncioLoop)
+ self.addCleanup(test._tearDownAsyncioRunner)
try:
test.debug()
except MyException:
@@ -360,6 +468,21 @@
test.doCleanups()
self.assertEqual(events, ['asyncSetUp', 'test', 'cleanup'])
+ def test_setup_get_event_loop(self):
+ # See https://github.com/python/cpython/issues/95736
+ # Make sure the default event loop is not used
+ asyncio.set_event_loop(None)
+
+ class TestCase1(unittest.IsolatedAsyncioTestCase):
+ def setUp(self):
+ asyncio.get_event_loop_policy().get_event_loop()
+
+ async def test_demo1(self):
+ pass
+
+ test = TestCase1('test_demo1')
+ result = test.run()
+ self.assertTrue(result.wasSuccessful())
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/unittest/test/test_break.py b/common/py3-stdlib/unittest/test/test_break.py
index eebd2b6..33cbdd2 100644
--- a/common/py3-stdlib/unittest/test/test_break.py
+++ b/common/py3-stdlib/unittest/test/test_break.py
@@ -4,14 +4,18 @@
import sys
import signal
import weakref
-
import unittest
+from test import support
+
@unittest.skipUnless(hasattr(os, 'kill'), "Test requires os.kill")
@unittest.skipIf(sys.platform =="win32", "Test cannot run on Windows")
class TestBreak(unittest.TestCase):
int_handler = None
+ # This number was smart-guessed, previously tests were failing
+ # after 7th run. So, we take `x * 2 + 1` to be sure.
+ default_repeats = 15
def setUp(self):
self._default_handler = signal.getsignal(signal.SIGINT)
@@ -24,6 +28,27 @@
unittest.signals._interrupt_handler = None
+ def withRepeats(self, test_function, repeats=None):
+ if not support.check_impl_detail(cpython=True):
+ # Override repeats count on non-cpython to execute only once.
+ # Because this test only makes sense to be repeated on CPython.
+ repeats = 1
+ elif repeats is None:
+ repeats = self.default_repeats
+
+ for repeat in range(repeats):
+ with self.subTest(repeat=repeat):
+ # We don't run `setUp` for the very first repeat
+ # and we don't run `tearDown` for the very last one,
+ # because they are handled by the test class itself.
+ if repeat != 0:
+ self.setUp()
+ try:
+ test_function()
+ finally:
+ if repeat != repeats - 1:
+ self.tearDown()
+
def testInstallHandler(self):
default_handler = signal.getsignal(signal.SIGINT)
unittest.installHandler()
@@ -48,35 +73,34 @@
unittest.removeResult(result)
def testInterruptCaught(self):
- default_handler = signal.getsignal(signal.SIGINT)
-
- result = unittest.TestResult()
- unittest.installHandler()
- unittest.registerResult(result)
-
- self.assertNotEqual(signal.getsignal(signal.SIGINT), default_handler)
-
def test(result):
pid = os.getpid()
os.kill(pid, signal.SIGINT)
result.breakCaught = True
self.assertTrue(result.shouldStop)
- try:
- test(result)
- except KeyboardInterrupt:
- self.fail("KeyboardInterrupt not handled")
- self.assertTrue(result.breakCaught)
+ def test_function():
+ result = unittest.TestResult()
+ unittest.installHandler()
+ unittest.registerResult(result)
+ self.assertNotEqual(
+ signal.getsignal(signal.SIGINT),
+ self._default_handler,
+ )
+
+ try:
+ test(result)
+ except KeyboardInterrupt:
+ self.fail("KeyboardInterrupt not handled")
+ self.assertTrue(result.breakCaught)
+ self.withRepeats(test_function)
def testSecondInterrupt(self):
# Can't use skipIf decorator because the signal handler may have
# been changed after defining this method.
if signal.getsignal(signal.SIGINT) == signal.SIG_IGN:
self.skipTest("test requires SIGINT to not be ignored")
- result = unittest.TestResult()
- unittest.installHandler()
- unittest.registerResult(result)
def test(result):
pid = os.getpid()
@@ -86,40 +110,40 @@
os.kill(pid, signal.SIGINT)
self.fail("Second KeyboardInterrupt not raised")
- try:
- test(result)
- except KeyboardInterrupt:
- pass
- else:
- self.fail("Second KeyboardInterrupt not raised")
- self.assertTrue(result.breakCaught)
+ def test_function():
+ result = unittest.TestResult()
+ unittest.installHandler()
+ unittest.registerResult(result)
+
+ with self.assertRaises(KeyboardInterrupt):
+ test(result)
+ self.assertTrue(result.breakCaught)
+ self.withRepeats(test_function)
def testTwoResults(self):
- unittest.installHandler()
+ def test_function():
+ unittest.installHandler()
- result = unittest.TestResult()
- unittest.registerResult(result)
- new_handler = signal.getsignal(signal.SIGINT)
+ result = unittest.TestResult()
+ unittest.registerResult(result)
+ new_handler = signal.getsignal(signal.SIGINT)
- result2 = unittest.TestResult()
- unittest.registerResult(result2)
- self.assertEqual(signal.getsignal(signal.SIGINT), new_handler)
+ result2 = unittest.TestResult()
+ unittest.registerResult(result2)
+ self.assertEqual(signal.getsignal(signal.SIGINT), new_handler)
- result3 = unittest.TestResult()
+ result3 = unittest.TestResult()
- def test(result):
- pid = os.getpid()
- os.kill(pid, signal.SIGINT)
+ try:
+ os.kill(os.getpid(), signal.SIGINT)
+ except KeyboardInterrupt:
+ self.fail("KeyboardInterrupt not handled")
- try:
- test(result)
- except KeyboardInterrupt:
- self.fail("KeyboardInterrupt not handled")
-
- self.assertTrue(result.shouldStop)
- self.assertTrue(result2.shouldStop)
- self.assertFalse(result3.shouldStop)
+ self.assertTrue(result.shouldStop)
+ self.assertTrue(result2.shouldStop)
+ self.assertFalse(result3.shouldStop)
+ self.withRepeats(test_function)
def testHandlerReplacedButCalled(self):
@@ -127,23 +151,25 @@
# been changed after defining this method.
if signal.getsignal(signal.SIGINT) == signal.SIG_IGN:
self.skipTest("test requires SIGINT to not be ignored")
- # If our handler has been replaced (is no longer installed) but is
- # called by the *new* handler, then it isn't safe to delay the
- # SIGINT and we should immediately delegate to the default handler
- unittest.installHandler()
- handler = signal.getsignal(signal.SIGINT)
- def new_handler(frame, signum):
- handler(frame, signum)
- signal.signal(signal.SIGINT, new_handler)
+ def test_function():
+ # If our handler has been replaced (is no longer installed) but is
+ # called by the *new* handler, then it isn't safe to delay the
+ # SIGINT and we should immediately delegate to the default handler
+ unittest.installHandler()
- try:
- pid = os.getpid()
- os.kill(pid, signal.SIGINT)
- except KeyboardInterrupt:
- pass
- else:
- self.fail("replaced but delegated handler doesn't raise interrupt")
+ handler = signal.getsignal(signal.SIGINT)
+ def new_handler(frame, signum):
+ handler(frame, signum)
+ signal.signal(signal.SIGINT, new_handler)
+
+ try:
+ os.kill(os.getpid(), signal.SIGINT)
+ except KeyboardInterrupt:
+ pass
+ else:
+ self.fail("replaced but delegated handler doesn't raise interrupt")
+ self.withRepeats(test_function)
def testRunner(self):
# Creating a TextTestRunner with the appropriate argument should
diff --git a/common/py3-stdlib/unittest/test/test_case.py b/common/py3-stdlib/unittest/test/test_case.py
index 9b3a598..78303b3 100644
--- a/common/py3-stdlib/unittest/test/test_case.py
+++ b/common/py3-stdlib/unittest/test/test_case.py
@@ -197,8 +197,8 @@
super(Foo, self).test()
raise RuntimeError('raised by Foo.test')
- expected = ['startTest', 'setUp', 'test', 'tearDown',
- 'addError', 'stopTest']
+ expected = ['startTest', 'setUp', 'test',
+ 'addError', 'tearDown', 'stopTest']
Foo(events).run(result)
self.assertEqual(events, expected)
@@ -216,7 +216,7 @@
raise RuntimeError('raised by Foo.test')
expected = ['startTestRun', 'startTest', 'setUp', 'test',
- 'tearDown', 'addError', 'stopTest', 'stopTestRun']
+ 'addError', 'tearDown', 'stopTest', 'stopTestRun']
Foo(events).run()
self.assertEqual(events, expected)
@@ -236,8 +236,8 @@
super(Foo, self).test()
self.fail('raised by Foo.test')
- expected = ['startTest', 'setUp', 'test', 'tearDown',
- 'addFailure', 'stopTest']
+ expected = ['startTest', 'setUp', 'test',
+ 'addFailure', 'tearDown', 'stopTest']
Foo(events).run(result)
self.assertEqual(events, expected)
@@ -252,7 +252,7 @@
self.fail('raised by Foo.test')
expected = ['startTestRun', 'startTest', 'setUp', 'test',
- 'tearDown', 'addFailure', 'stopTest', 'stopTestRun']
+ 'addFailure', 'tearDown', 'stopTest', 'stopTestRun']
events = []
Foo(events).run()
self.assertEqual(events, expected)
@@ -306,6 +306,37 @@
Foo('test').run()
+ def test_deprecation_of_return_val_from_test(self):
+ # Issue 41322 - deprecate return of value that is not None from a test
+ class Nothing:
+ def __eq__(self, o):
+ return o is None
+ class Foo(unittest.TestCase):
+ def test1(self):
+ return 1
+ def test2(self):
+ yield 1
+ def test3(self):
+ return Nothing()
+
+ with self.assertWarns(DeprecationWarning) as w:
+ Foo('test1').run()
+ self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
+ self.assertIn('test1', str(w.warning))
+ self.assertEqual(w.filename, __file__)
+
+ with self.assertWarns(DeprecationWarning) as w:
+ Foo('test2').run()
+ self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
+ self.assertIn('test2', str(w.warning))
+ self.assertEqual(w.filename, __file__)
+
+ with self.assertWarns(DeprecationWarning) as w:
+ Foo('test3').run()
+ self.assertIn('It is deprecated to return a value that is not None', str(w.warning))
+ self.assertIn('test3', str(w.warning))
+ self.assertEqual(w.filename, __file__)
+
def _check_call_order__subtests(self, result, events, expected_events):
class Foo(Test.LoggingTestCase):
def test(self):
@@ -333,10 +364,10 @@
def test_run_call_order__subtests(self):
events = []
result = LoggingResult(events)
- expected = ['startTest', 'setUp', 'test', 'tearDown',
+ expected = ['startTest', 'setUp', 'test',
'addSubTestFailure', 'addSubTestSuccess',
'addSubTestFailure', 'addSubTestFailure',
- 'addSubTestSuccess', 'addError', 'stopTest']
+ 'addSubTestSuccess', 'addError', 'tearDown', 'stopTest']
self._check_call_order__subtests(result, events, expected)
def test_run_call_order__subtests_legacy(self):
@@ -344,8 +375,8 @@
# text execution stops after the first subtest failure.
events = []
result = LegacyLoggingResult(events)
- expected = ['startTest', 'setUp', 'test', 'tearDown',
- 'addFailure', 'stopTest']
+ expected = ['startTest', 'setUp', 'test',
+ 'addFailure', 'tearDown', 'stopTest']
self._check_call_order__subtests(result, events, expected)
def _check_call_order__subtests_success(self, result, events, expected_events):
@@ -366,9 +397,9 @@
result = LoggingResult(events)
# The 6 subtest successes are individually recorded, in addition
# to the whole test success.
- expected = (['startTest', 'setUp', 'test', 'tearDown']
+ expected = (['startTest', 'setUp', 'test']
+ 6 * ['addSubTestSuccess']
- + ['addSuccess', 'stopTest'])
+ + ['tearDown', 'addSuccess', 'stopTest'])
self._check_call_order__subtests_success(result, events, expected)
def test_run_call_order__subtests_success_legacy(self):
@@ -393,8 +424,8 @@
self.fail('failure')
self.fail('failure')
- expected = ['startTest', 'setUp', 'test', 'tearDown',
- 'addSubTestFailure', 'stopTest']
+ expected = ['startTest', 'setUp', 'test',
+ 'addSubTestFailure', 'tearDown', 'stopTest']
Foo(events).run(result)
self.assertEqual(events, expected)
@@ -420,7 +451,7 @@
result = unittest.TestResult()
result.failfast = True
- suite = unittest.makeSuite(Foo)
+ suite = unittest.TestLoader().loadTestsFromTestCase(Foo)
suite.run(result)
expected = ['a1', 'a2', 'b1']
@@ -708,10 +739,6 @@
with self.assertRaises(self.failureException):
self.assertDictContainsSubset({'foo': one}, {'foo': '\uFFFD'})
- with self.assertWarns(DeprecationWarning) as warninfo:
- self.assertDictContainsSubset({}, {})
- self.assertEqual(warninfo.warnings[0].filename, __file__)
-
def testAssertEqual(self):
equal_pairs = [
((), ()),
diff --git a/common/py3-stdlib/unittest/test/test_discovery.py b/common/py3-stdlib/unittest/test/test_discovery.py
index 9d502c5..3b58786 100644
--- a/common/py3-stdlib/unittest/test/test_discovery.py
+++ b/common/py3-stdlib/unittest/test/test_discovery.py
@@ -396,7 +396,7 @@
self.addCleanup(restore_isdir)
_find_tests_args = []
- def _find_tests(start_dir, pattern, namespace=None):
+ def _find_tests(start_dir, pattern):
_find_tests_args.append((start_dir, pattern))
return ['tests']
loader._find_tests = _find_tests
@@ -792,7 +792,7 @@
expectedPath = os.path.abspath(os.path.dirname(unittest.test.__file__))
self.wasRun = False
- def _find_tests(start_dir, pattern, namespace=None):
+ def _find_tests(start_dir, pattern):
self.wasRun = True
self.assertEqual(start_dir, expectedPath)
return tests
@@ -825,37 +825,6 @@
'Can not use builtin modules '
'as dotted module names')
- def test_discovery_from_dotted_namespace_packages(self):
- loader = unittest.TestLoader()
-
- package = types.ModuleType('package')
- package.__path__ = ['/a', '/b']
- package.__spec__ = types.SimpleNamespace(
- loader=None,
- submodule_search_locations=['/a', '/b']
- )
-
- def _import(packagename, *args, **kwargs):
- sys.modules[packagename] = package
- return package
-
- _find_tests_args = []
- def _find_tests(start_dir, pattern, namespace=None):
- _find_tests_args.append((start_dir, pattern))
- return ['%s/tests' % start_dir]
-
- loader._find_tests = _find_tests
- loader.suiteClass = list
-
- with unittest.mock.patch('builtins.__import__', _import):
- # Since loader.discover() can modify sys.path, restore it when done.
- with import_helper.DirsOnSysPath():
- # Make sure to remove 'package' from sys.modules when done.
- with test.test_importlib.util.uncache('package'):
- suite = loader.discover('package')
-
- self.assertEqual(suite, ['/a/tests', '/b/tests'])
-
def test_discovery_failed_discovery(self):
loader = unittest.TestLoader()
package = types.ModuleType('package')
diff --git a/common/py3-stdlib/unittest/test/test_functiontestcase.py b/common/py3-stdlib/unittest/test/test_functiontestcase.py
index c5f2bcb..4971729 100644
--- a/common/py3-stdlib/unittest/test/test_functiontestcase.py
+++ b/common/py3-stdlib/unittest/test/test_functiontestcase.py
@@ -58,8 +58,8 @@
def tearDown():
events.append('tearDown')
- expected = ['startTest', 'setUp', 'test', 'tearDown',
- 'addError', 'stopTest']
+ expected = ['startTest', 'setUp', 'test',
+ 'addError', 'tearDown', 'stopTest']
unittest.FunctionTestCase(test, setUp, tearDown).run(result)
self.assertEqual(events, expected)
@@ -84,8 +84,8 @@
def tearDown():
events.append('tearDown')
- expected = ['startTest', 'setUp', 'test', 'tearDown',
- 'addFailure', 'stopTest']
+ expected = ['startTest', 'setUp', 'test',
+ 'addFailure', 'tearDown', 'stopTest']
unittest.FunctionTestCase(test, setUp, tearDown).run(result)
self.assertEqual(events, expected)
diff --git a/common/py3-stdlib/unittest/test/test_loader.py b/common/py3-stdlib/unittest/test/test_loader.py
index bc54bf0..de2268c 100644
--- a/common/py3-stdlib/unittest/test/test_loader.py
+++ b/common/py3-stdlib/unittest/test/test_loader.py
@@ -1591,5 +1591,52 @@
self.assertEqual(loader.getTestCaseNames(Foo), test_names)
+class TestObsoleteFunctions(unittest.TestCase):
+ class MyTestSuite(unittest.TestSuite):
+ pass
+
+ class MyTestCase(unittest.TestCase):
+ def check_1(self): pass
+ def check_2(self): pass
+ def test(self): pass
+
+ @staticmethod
+ def reverse_three_way_cmp(a, b):
+ return unittest.util.three_way_cmp(b, a)
+
+ def test_getTestCaseNames(self):
+ with self.assertWarns(DeprecationWarning) as w:
+ tests = unittest.getTestCaseNames(self.MyTestCase,
+ prefix='check', sortUsing=self.reverse_three_way_cmp,
+ testNamePatterns=None)
+ self.assertEqual(w.filename, __file__)
+ self.assertEqual(tests, ['check_2', 'check_1'])
+
+ def test_makeSuite(self):
+ with self.assertWarns(DeprecationWarning) as w:
+ suite = unittest.makeSuite(self.MyTestCase,
+ prefix='check', sortUsing=self.reverse_three_way_cmp,
+ suiteClass=self.MyTestSuite)
+ self.assertEqual(w.filename, __file__)
+ self.assertIsInstance(suite, self.MyTestSuite)
+ expected = self.MyTestSuite([self.MyTestCase('check_2'),
+ self.MyTestCase('check_1')])
+ self.assertEqual(suite, expected)
+
+ def test_findTestCases(self):
+ m = types.ModuleType('m')
+ m.testcase_1 = self.MyTestCase
+
+ with self.assertWarns(DeprecationWarning) as w:
+ suite = unittest.findTestCases(m,
+ prefix='check', sortUsing=self.reverse_three_way_cmp,
+ suiteClass=self.MyTestSuite)
+ self.assertEqual(w.filename, __file__)
+ self.assertIsInstance(suite, self.MyTestSuite)
+ expected = [self.MyTestSuite([self.MyTestCase('check_2'),
+ self.MyTestCase('check_1')])]
+ self.assertEqual(list(suite), expected)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/common/py3-stdlib/unittest/test/test_program.py b/common/py3-stdlib/unittest/test/test_program.py
index b7fbbc1..26a8550 100644
--- a/common/py3-stdlib/unittest/test/test_program.py
+++ b/common/py3-stdlib/unittest/test/test_program.py
@@ -61,6 +61,17 @@
pass
def testFail(self):
raise AssertionError
+ def testError(self):
+ 1/0
+ @unittest.skip('skipping')
+ def testSkipped(self):
+ raise AssertionError
+ @unittest.expectedFailure
+ def testExpectedFailure(self):
+ raise AssertionError
+ @unittest.expectedFailure
+ def testUnexpectedSuccess(self):
+ pass
class FooBarLoader(unittest.TestLoader):
"""Test loader that returns a suite containing FooBar."""
@@ -111,9 +122,13 @@
testRunner=unittest.TextTestRunner(stream=stream),
testLoader=self.FooBarLoader())
self.assertTrue(hasattr(program, 'result'))
- self.assertIn('\nFAIL: testFail ', stream.getvalue())
- self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
-
+ out = stream.getvalue()
+ self.assertIn('\nFAIL: testFail ', out)
+ self.assertIn('\nERROR: testError ', out)
+ self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out)
+ expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, '
+ 'expected failures=1, unexpected successes=1)\n')
+ self.assertTrue(out.endswith(expected))
def test_Exit(self):
stream = BufferedWriter()
@@ -124,9 +139,13 @@
testRunner=unittest.TextTestRunner(stream=stream),
exit=True,
testLoader=self.FooBarLoader())
- self.assertIn('\nFAIL: testFail ', stream.getvalue())
- self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
-
+ out = stream.getvalue()
+ self.assertIn('\nFAIL: testFail ', out)
+ self.assertIn('\nERROR: testError ', out)
+ self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out)
+ expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, '
+ 'expected failures=1, unexpected successes=1)\n')
+ self.assertTrue(out.endswith(expected))
def test_ExitAsDefault(self):
stream = BufferedWriter()
@@ -136,8 +155,13 @@
argv=["foobar"],
testRunner=unittest.TextTestRunner(stream=stream),
testLoader=self.FooBarLoader())
- self.assertIn('\nFAIL: testFail ', stream.getvalue())
- self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
+ out = stream.getvalue()
+ self.assertIn('\nFAIL: testFail ', out)
+ self.assertIn('\nERROR: testError ', out)
+ self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out)
+ expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, '
+ 'expected failures=1, unexpected successes=1)\n')
+ self.assertTrue(out.endswith(expected))
class InitialisableProgram(unittest.TestProgram):
@@ -172,6 +196,7 @@
return RESULT
+@support.requires_subprocess()
class TestCommandLineArgs(unittest.TestCase):
def setUp(self):
@@ -429,7 +454,9 @@
def testSelectedTestNamesFunctionalTest(self):
def run_unittest(args):
- p = subprocess.Popen([sys.executable, '-m', 'unittest'] + args,
+ # Use -E to ignore PYTHONSAFEPATH env var
+ cmd = [sys.executable, '-E', '-m', 'unittest'] + args
+ p = subprocess.Popen(cmd,
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=os.path.dirname(__file__))
with p:
_, stderr = p.communicate()
diff --git a/common/py3-stdlib/unittest/test/test_result.py b/common/py3-stdlib/unittest/test/test_result.py
index 90484fd..9320b0a 100644
--- a/common/py3-stdlib/unittest/test/test_result.py
+++ b/common/py3-stdlib/unittest/test/test_result.py
@@ -432,115 +432,6 @@
self.assertIs(test_case, subtest)
self.assertIn("some recognizable failure", formatted_exc)
- def testGetDescriptionWithoutDocstring(self):
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self),
- 'testGetDescriptionWithoutDocstring (' + __name__ +
- '.Test_TestResult)')
-
- def testGetSubTestDescriptionWithoutDocstring(self):
- with self.subTest(foo=1, bar=2):
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self._subtest),
- 'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
- '.Test_TestResult) (foo=1, bar=2)')
- with self.subTest('some message'):
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self._subtest),
- 'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
- '.Test_TestResult) [some message]')
-
- def testGetSubTestDescriptionWithoutDocstringAndParams(self):
- with self.subTest():
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self._subtest),
- 'testGetSubTestDescriptionWithoutDocstringAndParams '
- '(' + __name__ + '.Test_TestResult) (<subtest>)')
-
- def testGetSubTestDescriptionForFalsyValues(self):
- expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TestResult) [%s]'
- result = unittest.TextTestResult(None, True, 1)
- for arg in [0, None, []]:
- with self.subTest(arg):
- self.assertEqual(
- result.getDescription(self._subtest),
- expected % (__name__, arg)
- )
-
- def testGetNestedSubTestDescriptionWithoutDocstring(self):
- with self.subTest(foo=1):
- with self.subTest(baz=2, bar=3):
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self._subtest),
- 'testGetNestedSubTestDescriptionWithoutDocstring '
- '(' + __name__ + '.Test_TestResult) (baz=2, bar=3, foo=1)')
-
- def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
- with self.subTest(foo=1, bar=2):
- with self.subTest(baz=3, bar=4):
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self._subtest),
- 'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
- '(' + __name__ + '.Test_TestResult) (baz=3, bar=4, foo=1)')
-
- @unittest.skipIf(sys.flags.optimize >= 2,
- "Docstrings are omitted with -O2 and above")
- def testGetDescriptionWithOneLineDocstring(self):
- """Tests getDescription() for a method with a docstring."""
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self),
- ('testGetDescriptionWithOneLineDocstring '
- '(' + __name__ + '.Test_TestResult)\n'
- 'Tests getDescription() for a method with a docstring.'))
-
- @unittest.skipIf(sys.flags.optimize >= 2,
- "Docstrings are omitted with -O2 and above")
- def testGetSubTestDescriptionWithOneLineDocstring(self):
- """Tests getDescription() for a method with a docstring."""
- result = unittest.TextTestResult(None, True, 1)
- with self.subTest(foo=1, bar=2):
- self.assertEqual(
- result.getDescription(self._subtest),
- ('testGetSubTestDescriptionWithOneLineDocstring '
- '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
- 'Tests getDescription() for a method with a docstring.'))
-
- @unittest.skipIf(sys.flags.optimize >= 2,
- "Docstrings are omitted with -O2 and above")
- def testGetDescriptionWithMultiLineDocstring(self):
- """Tests getDescription() for a method with a longer docstring.
- The second line of the docstring.
- """
- result = unittest.TextTestResult(None, True, 1)
- self.assertEqual(
- result.getDescription(self),
- ('testGetDescriptionWithMultiLineDocstring '
- '(' + __name__ + '.Test_TestResult)\n'
- 'Tests getDescription() for a method with a longer '
- 'docstring.'))
-
- @unittest.skipIf(sys.flags.optimize >= 2,
- "Docstrings are omitted with -O2 and above")
- def testGetSubTestDescriptionWithMultiLineDocstring(self):
- """Tests getDescription() for a method with a longer docstring.
- The second line of the docstring.
- """
- result = unittest.TextTestResult(None, True, 1)
- with self.subTest(foo=1, bar=2):
- self.assertEqual(
- result.getDescription(self._subtest),
- ('testGetSubTestDescriptionWithMultiLineDocstring '
- '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
- 'Tests getDescription() for a method with a longer '
- 'docstring.'))
-
def testStackFrameTrimming(self):
class Frame(object):
class tb_frame(object):
@@ -580,6 +471,239 @@
self.assertTrue(stream.getvalue().endswith('\n\nOK\n'))
+class Test_TextTestResult(unittest.TestCase):
+ maxDiff = None
+
+ def testGetDescriptionWithoutDocstring(self):
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self),
+ 'testGetDescriptionWithoutDocstring (' + __name__ +
+ '.Test_TextTestResult.testGetDescriptionWithoutDocstring)')
+
+ def testGetSubTestDescriptionWithoutDocstring(self):
+ with self.subTest(foo=1, bar=2):
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ 'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
+ '.Test_TextTestResult.testGetSubTestDescriptionWithoutDocstring) (foo=1, bar=2)')
+
+ with self.subTest('some message'):
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ 'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
+ '.Test_TextTestResult.testGetSubTestDescriptionWithoutDocstring) [some message]')
+
+ def testGetSubTestDescriptionWithoutDocstringAndParams(self):
+ with self.subTest():
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ 'testGetSubTestDescriptionWithoutDocstringAndParams '
+ '(' + __name__ + '.Test_TextTestResult.testGetSubTestDescriptionWithoutDocstringAndParams) '
+ '(<subtest>)')
+
+ def testGetSubTestDescriptionForFalsyValues(self):
+ expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TextTestResult.testGetSubTestDescriptionForFalsyValues) [%s]'
+ result = unittest.TextTestResult(None, True, 1)
+ for arg in [0, None, []]:
+ with self.subTest(arg):
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ expected % (__name__, arg)
+ )
+
+ def testGetNestedSubTestDescriptionWithoutDocstring(self):
+ with self.subTest(foo=1):
+ with self.subTest(baz=2, bar=3):
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ 'testGetNestedSubTestDescriptionWithoutDocstring '
+ '(' + __name__ + '.Test_TextTestResult.testGetNestedSubTestDescriptionWithoutDocstring) '
+ '(baz=2, bar=3, foo=1)')
+
+ def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
+ with self.subTest(foo=1, bar=2):
+ with self.subTest(baz=3, bar=4):
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ 'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
+ '(' + __name__ + '.Test_TextTestResult.testGetDuplicatedNestedSubTestDescriptionWithoutDocstring) (baz=3, bar=4, foo=1)')
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def testGetDescriptionWithOneLineDocstring(self):
+ """Tests getDescription() for a method with a docstring."""
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self),
+ ('testGetDescriptionWithOneLineDocstring '
+ '(' + __name__ + '.Test_TextTestResult.testGetDescriptionWithOneLineDocstring)\n'
+ 'Tests getDescription() for a method with a docstring.'))
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def testGetSubTestDescriptionWithOneLineDocstring(self):
+ """Tests getDescription() for a method with a docstring."""
+ result = unittest.TextTestResult(None, True, 1)
+ with self.subTest(foo=1, bar=2):
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ ('testGetSubTestDescriptionWithOneLineDocstring '
+ '(' + __name__ + '.Test_TextTestResult.testGetSubTestDescriptionWithOneLineDocstring) '
+ '(foo=1, bar=2)\n'
+
+ 'Tests getDescription() for a method with a docstring.'))
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def testGetDescriptionWithMultiLineDocstring(self):
+ """Tests getDescription() for a method with a longer docstring.
+ The second line of the docstring.
+ """
+ result = unittest.TextTestResult(None, True, 1)
+ self.assertEqual(
+ result.getDescription(self),
+ ('testGetDescriptionWithMultiLineDocstring '
+ '(' + __name__ + '.Test_TextTestResult.testGetDescriptionWithMultiLineDocstring)\n'
+ 'Tests getDescription() for a method with a longer '
+ 'docstring.'))
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def testGetSubTestDescriptionWithMultiLineDocstring(self):
+ """Tests getDescription() for a method with a longer docstring.
+ The second line of the docstring.
+ """
+ result = unittest.TextTestResult(None, True, 1)
+ with self.subTest(foo=1, bar=2):
+ self.assertEqual(
+ result.getDescription(self._subtest),
+ ('testGetSubTestDescriptionWithMultiLineDocstring '
+ '(' + __name__ + '.Test_TextTestResult.testGetSubTestDescriptionWithMultiLineDocstring) '
+ '(foo=1, bar=2)\n'
+ 'Tests getDescription() for a method with a longer '
+ 'docstring.'))
+
+ class Test(unittest.TestCase):
+ def testSuccess(self):
+ pass
+ def testSkip(self):
+ self.skipTest('skip')
+ def testFail(self):
+ self.fail('fail')
+ def testError(self):
+ raise Exception('error')
+ @unittest.expectedFailure
+ def testExpectedFailure(self):
+ self.fail('fail')
+ @unittest.expectedFailure
+ def testUnexpectedSuccess(self):
+ pass
+ def testSubTestSuccess(self):
+ with self.subTest('one', a=1):
+ pass
+ with self.subTest('two', b=2):
+ pass
+ def testSubTestMixed(self):
+ with self.subTest('success', a=1):
+ pass
+ with self.subTest('skip', b=2):
+ self.skipTest('skip')
+ with self.subTest('fail', c=3):
+ self.fail('fail')
+ with self.subTest('error', d=4):
+ raise Exception('error')
+
+ tearDownError = None
+ def tearDown(self):
+ if self.tearDownError is not None:
+ raise self.tearDownError
+
+ def _run_test(self, test_name, verbosity, tearDownError=None):
+ stream = BufferedWriter()
+ stream = unittest.runner._WritelnDecorator(stream)
+ result = unittest.TextTestResult(stream, True, verbosity)
+ test = self.Test(test_name)
+ test.tearDownError = tearDownError
+ test.run(result)
+ return stream.getvalue()
+
+ def testDotsOutput(self):
+ self.assertEqual(self._run_test('testSuccess', 1), '.')
+ self.assertEqual(self._run_test('testSkip', 1), 's')
+ self.assertEqual(self._run_test('testFail', 1), 'F')
+ self.assertEqual(self._run_test('testError', 1), 'E')
+ self.assertEqual(self._run_test('testExpectedFailure', 1), 'x')
+ self.assertEqual(self._run_test('testUnexpectedSuccess', 1), 'u')
+
+ def testLongOutput(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ self.assertEqual(self._run_test('testSuccess', 2),
+ f'testSuccess ({classname}.testSuccess) ... ok\n')
+ self.assertEqual(self._run_test('testSkip', 2),
+ f"testSkip ({classname}.testSkip) ... skipped 'skip'\n")
+ self.assertEqual(self._run_test('testFail', 2),
+ f'testFail ({classname}.testFail) ... FAIL\n')
+ self.assertEqual(self._run_test('testError', 2),
+ f'testError ({classname}.testError) ... ERROR\n')
+ self.assertEqual(self._run_test('testExpectedFailure', 2),
+ f'testExpectedFailure ({classname}.testExpectedFailure) ... expected failure\n')
+ self.assertEqual(self._run_test('testUnexpectedSuccess', 2),
+ f'testUnexpectedSuccess ({classname}.testUnexpectedSuccess) ... unexpected success\n')
+
+ def testDotsOutputSubTestSuccess(self):
+ self.assertEqual(self._run_test('testSubTestSuccess', 1), '.')
+
+ def testLongOutputSubTestSuccess(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ self.assertEqual(self._run_test('testSubTestSuccess', 2),
+ f'testSubTestSuccess ({classname}.testSubTestSuccess) ... ok\n')
+
+ def testDotsOutputSubTestMixed(self):
+ self.assertEqual(self._run_test('testSubTestMixed', 1), 'sFE')
+
+ def testLongOutputSubTestMixed(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ self.assertEqual(self._run_test('testSubTestMixed', 2),
+ f'testSubTestMixed ({classname}.testSubTestMixed) ... \n'
+ f" testSubTestMixed ({classname}.testSubTestMixed) [skip] (b=2) ... skipped 'skip'\n"
+ f' testSubTestMixed ({classname}.testSubTestMixed) [fail] (c=3) ... FAIL\n'
+ f' testSubTestMixed ({classname}.testSubTestMixed) [error] (d=4) ... ERROR\n')
+
+ def testDotsOutputTearDownFail(self):
+ out = self._run_test('testSuccess', 1, AssertionError('fail'))
+ self.assertEqual(out, 'F')
+ out = self._run_test('testError', 1, AssertionError('fail'))
+ self.assertEqual(out, 'EF')
+ out = self._run_test('testFail', 1, Exception('error'))
+ self.assertEqual(out, 'FE')
+ out = self._run_test('testSkip', 1, AssertionError('fail'))
+ self.assertEqual(out, 'sF')
+
+ def testLongOutputTearDownFail(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ out = self._run_test('testSuccess', 2, AssertionError('fail'))
+ self.assertEqual(out,
+ f'testSuccess ({classname}.testSuccess) ... FAIL\n')
+ out = self._run_test('testError', 2, AssertionError('fail'))
+ self.assertEqual(out,
+ f'testError ({classname}.testError) ... ERROR\n'
+ f'testError ({classname}.testError) ... FAIL\n')
+ out = self._run_test('testFail', 2, Exception('error'))
+ self.assertEqual(out,
+ f'testFail ({classname}.testFail) ... FAIL\n'
+ f'testFail ({classname}.testFail) ... ERROR\n')
+ out = self._run_test('testSkip', 2, AssertionError('fail'))
+ self.assertEqual(out,
+ f"testSkip ({classname}.testSkip) ... skipped 'skip'\n"
+ f'testSkip ({classname}.testSkip) ... FAIL\n')
+
+
classDict = dict(unittest.TestResult.__dict__)
for m in ('addSkip', 'addExpectedFailure', 'addUnexpectedSuccess',
'__init__'):
@@ -790,7 +914,7 @@
expected_out = '\nStdout:\nset up\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
- description = f'test_foo ({strclass(Foo)})'
+ description = f'test_foo ({strclass(Foo)}.test_foo)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
@@ -812,7 +936,7 @@
expected_out = '\nStdout:\ntear down\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 1)
- description = f'test_foo ({strclass(Foo)})'
+ description = f'test_foo ({strclass(Foo)}.test_foo)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
@@ -835,12 +959,13 @@
expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 2)
- description = f'test_foo ({strclass(Foo)})'
+ description = f'test_foo ({strclass(Foo)}.test_foo)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
- self.assertIn(expected_out, formatted_exc)
+ self.assertIn('\nStdout:\nset up\ndo cleanup2\n', formatted_exc)
+ self.assertNotIn('\ndo cleanup1\n', formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(str(test_case), description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
@@ -865,19 +990,22 @@
expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 3)
- description = f'test_foo ({strclass(Foo)})'
+ description = f'test_foo ({strclass(Foo)}.test_foo)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
- self.assertIn(expected_out, formatted_exc)
+ self.assertIn('\nStdout:\nset up\n', formatted_exc)
+ self.assertNotIn('\ndo cleanup2\n', formatted_exc)
+ self.assertNotIn('\ndo cleanup1\n', formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(str(test_case), description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
- self.assertIn(expected_out, formatted_exc)
+ self.assertIn('\nStdout:\nset up\ndo cleanup2\n', formatted_exc)
+ self.assertNotIn('\ndo cleanup1\n', formatted_exc)
test_case, formatted_exc = result.errors[2]
self.assertEqual(str(test_case), description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
@@ -905,19 +1033,22 @@
expected_out = '\nStdout:\nset up\ntear down\ndo cleanup2\ndo cleanup1\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(len(result.errors), 3)
- description = f'test_foo ({strclass(Foo)})'
+ description = f'test_foo ({strclass(Foo)}.test_foo)'
test_case, formatted_exc = result.errors[0]
self.assertEqual(str(test_case), description)
self.assertIn('ZeroDivisionError: division by zero', formatted_exc)
self.assertNotIn('ValueError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
- self.assertIn(expected_out, formatted_exc)
+ self.assertIn('\nStdout:\nset up\ntear down\n', formatted_exc)
+ self.assertNotIn('\ndo cleanup2\n', formatted_exc)
+ self.assertNotIn('\ndo cleanup1\n', formatted_exc)
test_case, formatted_exc = result.errors[1]
self.assertEqual(str(test_case), description)
self.assertIn('ValueError: bad cleanup2', formatted_exc)
self.assertNotIn('ZeroDivisionError', formatted_exc)
self.assertNotIn('TypeError', formatted_exc)
- self.assertIn(expected_out, formatted_exc)
+ self.assertIn('\nStdout:\nset up\ntear down\ndo cleanup2\n', formatted_exc)
+ self.assertNotIn('\ndo cleanup1\n', formatted_exc)
test_case, formatted_exc = result.errors[2]
self.assertEqual(str(test_case), description)
self.assertIn('TypeError: bad cleanup1', formatted_exc)
diff --git a/common/py3-stdlib/unittest/test/test_runner.py b/common/py3-stdlib/unittest/test/test_runner.py
index 0082d39..e02b2da 100644
--- a/common/py3-stdlib/unittest/test/test_runner.py
+++ b/common/py3-stdlib/unittest/test/test_runner.py
@@ -3,6 +3,7 @@
import sys
import pickle
import subprocess
+from test import support
import unittest
from unittest.case import _Outcome
@@ -45,6 +46,29 @@
raise Exception('CleanUpExc')
+class TestCM:
+ def __init__(self, ordering, enter_result=None):
+ self.ordering = ordering
+ self.enter_result = enter_result
+
+ def __enter__(self):
+ self.ordering.append('enter')
+ return self.enter_result
+
+ def __exit__(self, *exc_info):
+ self.ordering.append('exit')
+
+
+class LacksEnterAndExit:
+ pass
+class LacksEnter:
+ def __exit__(self, *exc_info):
+ pass
+class LacksExit:
+ def __enter__(self):
+ pass
+
+
class TestCleanUp(unittest.TestCase):
def testCleanUp(self):
class TestableTest(unittest.TestCase):
@@ -78,7 +102,8 @@
pass
test = TestableTest('testNothing')
- outcome = test._outcome = _Outcome()
+ result = unittest.TestResult()
+ outcome = test._outcome = _Outcome(result=result)
CleanUpExc = Exception('foo')
exc2 = Exception('bar')
@@ -94,10 +119,13 @@
self.assertFalse(test.doCleanups())
self.assertFalse(outcome.success)
- ((_, (Type1, instance1, _)),
- (_, (Type2, instance2, _))) = reversed(outcome.errors)
- self.assertEqual((Type1, instance1), (Exception, CleanUpExc))
- self.assertEqual((Type2, instance2), (Exception, exc2))
+ (_, msg2), (_, msg1) = result.errors
+ self.assertIn('in cleanup1', msg1)
+ self.assertIn('raise CleanUpExc', msg1)
+ self.assertIn('Exception: foo', msg1)
+ self.assertIn('in cleanup2', msg2)
+ self.assertIn('raise exc2', msg2)
+ self.assertIn('Exception: bar', msg2)
def testCleanupInRun(self):
blowUp = False
@@ -178,6 +206,39 @@
'cleanup3', 'cleanup1', 'cleanup2'])
+ def test_enterContext(self):
+ class TestableTest(unittest.TestCase):
+ def testNothing(self):
+ pass
+
+ test = TestableTest('testNothing')
+ cleanups = []
+
+ test.addCleanup(cleanups.append, 'cleanup1')
+ cm = TestCM(cleanups, 42)
+ self.assertEqual(test.enterContext(cm), 42)
+ test.addCleanup(cleanups.append, 'cleanup2')
+
+ self.assertTrue(test.doCleanups())
+ self.assertEqual(cleanups, ['enter', 'cleanup2', 'exit', 'cleanup1'])
+
+ def test_enterContext_arg_errors(self):
+ class TestableTest(unittest.TestCase):
+ def testNothing(self):
+ pass
+
+ test = TestableTest('testNothing')
+
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ test.enterContext(LacksEnterAndExit())
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ test.enterContext(LacksEnter())
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ test.enterContext(LacksExit())
+
+ self.assertEqual(test._cleanups, [])
+
+
class TestClassCleanup(unittest.TestCase):
def test_addClassCleanUp(self):
class TestableTest(unittest.TestCase):
@@ -457,6 +518,35 @@
self.assertEqual(ordering,
['setUpClass', 'test', 'tearDownClass', 'cleanup_good'])
+ def test_enterClassContext(self):
+ class TestableTest(unittest.TestCase):
+ def testNothing(self):
+ pass
+
+ cleanups = []
+
+ TestableTest.addClassCleanup(cleanups.append, 'cleanup1')
+ cm = TestCM(cleanups, 42)
+ self.assertEqual(TestableTest.enterClassContext(cm), 42)
+ TestableTest.addClassCleanup(cleanups.append, 'cleanup2')
+
+ TestableTest.doClassCleanups()
+ self.assertEqual(cleanups, ['enter', 'cleanup2', 'exit', 'cleanup1'])
+
+ def test_enterClassContext_arg_errors(self):
+ class TestableTest(unittest.TestCase):
+ def testNothing(self):
+ pass
+
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ TestableTest.enterClassContext(LacksEnterAndExit())
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ TestableTest.enterClassContext(LacksEnter())
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ TestableTest.enterClassContext(LacksExit())
+
+ self.assertEqual(TestableTest._class_cleanups, [])
+
def test_run_nested_test(self):
ordering = []
@@ -1036,6 +1126,31 @@
'cleanup2', 'setUp2', 'test2', 'tearDown2',
'cleanup3', 'tearDownModule', 'cleanup1'])
+ def test_enterModuleContext(self):
+ cleanups = []
+
+ unittest.addModuleCleanup(cleanups.append, 'cleanup1')
+ cm = TestCM(cleanups, 42)
+ self.assertEqual(unittest.enterModuleContext(cm), 42)
+ unittest.addModuleCleanup(cleanups.append, 'cleanup2')
+
+ unittest.case.doModuleCleanups()
+ self.assertEqual(cleanups, ['enter', 'cleanup2', 'exit', 'cleanup1'])
+
+ def test_enterModuleContext_arg_errors(self):
+ class TestableTest(unittest.TestCase):
+ def testNothing(self):
+ pass
+
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ unittest.enterModuleContext(LacksEnterAndExit())
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ unittest.enterModuleContext(LacksEnter())
+ with self.assertRaisesRegex(TypeError, 'the context manager'):
+ unittest.enterModuleContext(LacksExit())
+
+ self.assertEqual(unittest.case._module_cleanups, [])
+
class Test_TextTestRunner(unittest.TestCase):
"""Tests for TextTestRunner."""
@@ -1176,6 +1291,7 @@
expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY)
self.assertEqual(runner._makeResult(), expectedresult)
+ @support.requires_subprocess()
def test_warnings(self):
"""
Check that warnings argument of TextTestRunner correctly affects the
diff --git a/common/py3-stdlib/unittest/test/test_skipping.py b/common/py3-stdlib/unittest/test/test_skipping.py
index 7cb9d33..64ceeae 100644
--- a/common/py3-stdlib/unittest/test/test_skipping.py
+++ b/common/py3-stdlib/unittest/test/test_skipping.py
@@ -197,7 +197,7 @@
result = LoggingResult(events)
test = Foo("test_skip_me")
self.assertIs(test.run(result), result)
- self.assertEqual(events, ['startTest', 'addSkip', 'addFailure', 'stopTest'])
+ self.assertEqual(events, ['startTest', 'addFailure', 'addSkip', 'stopTest'])
self.assertEqual(result.skipped, [(test, "skip")])
def test_skipping_and_fail_in_cleanup(self):
diff --git a/common/py3-stdlib/unittest/test/testmock/testasync.py b/common/py3-stdlib/unittest/test/testmock/testasync.py
index 61269d6..df260ab 100644
--- a/common/py3-stdlib/unittest/test/testmock/testasync.py
+++ b/common/py3-stdlib/unittest/test/testmock/testasync.py
@@ -4,6 +4,9 @@
import re
import unittest
from contextlib import contextmanager
+from test import support
+
+support.requires_working_socket(module=True)
from asyncio import run, iscoroutinefunction
from unittest import IsolatedAsyncioTestCase
@@ -190,8 +193,7 @@
def test_future_isfuture(self):
loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- fut = asyncio.Future()
+ fut = loop.create_future()
loop.stop()
loop.close()
mock = AsyncMock(fut)
diff --git a/common/py3-stdlib/unittest/test/testmock/testhelpers.py b/common/py3-stdlib/unittest/test/testmock/testhelpers.py
index 9e7ec5d..74785a8 100644
--- a/common/py3-stdlib/unittest/test/testmock/testhelpers.py
+++ b/common/py3-stdlib/unittest/test/testmock/testhelpers.py
@@ -952,6 +952,24 @@
self.assertFalse(hasattr(autospec, '__name__'))
+ def test_autospec_signature_staticmethod(self):
+ class Foo:
+ @staticmethod
+ def static_method(a, b=10, *, c): pass
+
+ mock = create_autospec(Foo.__dict__['static_method'])
+ self.assertEqual(inspect.signature(Foo.static_method), inspect.signature(mock))
+
+
+ def test_autospec_signature_classmethod(self):
+ class Foo:
+ @classmethod
+ def class_method(cls, a, b=10, *, c): pass
+
+ mock = create_autospec(Foo.__dict__['class_method'])
+ self.assertEqual(inspect.signature(Foo.class_method), inspect.signature(mock))
+
+
def test_spec_inspect_signature(self):
def myfunc(x, y): pass
@@ -1077,7 +1095,7 @@
p.stop()
- def test_propertymock_returnvalue(self):
+ def test_propertymock_bare(self):
m = MagicMock()
p = PropertyMock()
type(m).foo = p
@@ -1088,6 +1106,27 @@
self.assertNotIsInstance(returned, PropertyMock)
+ def test_propertymock_returnvalue(self):
+ m = MagicMock()
+ p = PropertyMock(return_value=42)
+ type(m).foo = p
+
+ returned = m.foo
+ p.assert_called_once_with()
+ self.assertEqual(returned, 42)
+ self.assertNotIsInstance(returned, PropertyMock)
+
+
+ def test_propertymock_side_effect(self):
+ m = MagicMock()
+ p = PropertyMock(side_effect=ValueError)
+ type(m).foo = p
+
+ with self.assertRaises(ValueError):
+ m.foo
+ p.assert_called_once_with()
+
+
class TestCallablePredicate(unittest.TestCase):
def test_type(self):
diff --git a/common/py3-stdlib/unittest/test/testmock/testmock.py b/common/py3-stdlib/unittest/test/testmock/testmock.py
index 535f908..eaae22e 100644
--- a/common/py3-stdlib/unittest/test/testmock/testmock.py
+++ b/common/py3-stdlib/unittest/test/testmock/testmock.py
@@ -226,6 +226,14 @@
with self.assertRaisesRegex(InvalidSpecError,
"Cannot spec attr 'B' as the spec_set "):
mock.patch.object(A, 'B', spec_set=A.B).start()
+ with self.assertRaisesRegex(InvalidSpecError,
+ "Cannot spec attr 'B' as the spec_set "):
+ mock.patch.object(A, 'B', spec_set=A.B).start()
+ with self.assertRaisesRegex(InvalidSpecError, "Cannot spec a Mock object."):
+ mock.Mock(A.B)
+ with mock.patch('builtins.open', mock.mock_open()):
+ mock.mock_open() # should still be valid with open() mocked
+
def test_reset_mock(self):
parent = Mock()
diff --git a/common/py3-stdlib/unittest/test/testmock/testpatch.py b/common/py3-stdlib/unittest/test/testmock/testpatch.py
index 8ab63a1..cda286b 100644
--- a/common/py3-stdlib/unittest/test/testmock/testpatch.py
+++ b/common/py3-stdlib/unittest/test/testmock/testpatch.py
@@ -996,6 +996,36 @@
method.assert_called_once_with()
+ def test_autospec_staticmethod_signature(self):
+ # Patched methods which are decorated with @staticmethod should have the same signature
+ class Foo:
+ @staticmethod
+ def static_method(a, b=10, *, c): pass
+
+ Foo.static_method(1, 2, c=3)
+
+ with patch.object(Foo, 'static_method', autospec=True) as method:
+ method(1, 2, c=3)
+ self.assertRaises(TypeError, method)
+ self.assertRaises(TypeError, method, 1)
+ self.assertRaises(TypeError, method, 1, 2, 3, c=4)
+
+
+ def test_autospec_classmethod_signature(self):
+ # Patched methods which are decorated with @classmethod should have the same signature
+ class Foo:
+ @classmethod
+ def class_method(cls, a, b=10, *, c): pass
+
+ Foo.class_method(1, 2, c=3)
+
+ with patch.object(Foo, 'class_method', autospec=True) as method:
+ method(1, 2, c=3)
+ self.assertRaises(TypeError, method)
+ self.assertRaises(TypeError, method, 1)
+ self.assertRaises(TypeError, method, 1, 2, 3, c=4)
+
+
def test_autospec_with_new(self):
patcher = patch('%s.function' % __name__, new=3, autospec=True)
self.assertRaises(TypeError, patcher.start)
diff --git a/common/py3-stdlib/unittest/test/testmock/testwith.py b/common/py3-stdlib/unittest/test/testmock/testwith.py
index 42ebf38..c74d49a 100644
--- a/common/py3-stdlib/unittest/test/testmock/testwith.py
+++ b/common/py3-stdlib/unittest/test/testmock/testwith.py
@@ -130,8 +130,8 @@
c = C()
- with patch.object(c, 'f', autospec=True) as patch1:
- with patch.object(c, 'f', autospec=True) as patch2:
+ with patch.object(c, 'f') as patch1:
+ with patch.object(c, 'f') as patch2:
c.f()
self.assertEqual(patch2.call_count, 1)
self.assertEqual(patch1.call_count, 0)
diff --git a/common/py3-stdlib/urllib/parse.py b/common/py3-stdlib/urllib/parse.py
index 26ddf30..e5f0b78 100644
--- a/common/py3-stdlib/urllib/parse.py
+++ b/common/py3-stdlib/urllib/parse.py
@@ -25,13 +25,19 @@
scenarios for parsing, and for backward compatibility purposes, some
parsing quirks from older RFCs are retained. The testcases in
test_urlparse.py provides a good indicator of parsing behavior.
+
+The WHATWG URL Parser spec should also be considered. We are not compliant with
+it either due to existing user code API behavior expectations (Hyrum's Law).
+It serves as a useful guide when making changes.
"""
+from collections import namedtuple
+import functools
import re
import sys
import types
-import collections
import warnings
+import ipaddress
__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
"urlsplit", "urlunsplit", "urlencode", "parse_qs",
@@ -78,18 +84,17 @@
'0123456789'
'+-.')
+# Leading and trailing C0 control and space to be stripped per WHATWG spec.
+# == "".join([chr(i) for i in range(0, 0x20 + 1)])
+_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f '
+
# Unsafe bytes to be removed per WHATWG spec
_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
-# XXX: Consider replacing with functools.lru_cache
-MAX_CACHE_SIZE = 20
-_parse_cache = {}
-
def clear_cache():
- """Clear the parse cache and the quoters cache."""
- _parse_cache.clear()
- _safe_quoters.clear()
-
+ """Clear internal performance caches. Undocumented; some tests want it."""
+ urlsplit.cache_clear()
+ _byte_quoter_factory.cache_clear()
# Helpers for bytes handling
# For 3.2, we deliberately require applications that
@@ -242,8 +247,6 @@
return hostname, port
-from collections import namedtuple
-
_DefragResultBase = namedtuple('DefragResult', 'url fragment')
_SplitResultBase = namedtuple(
'SplitResult', 'scheme netloc path query fragment')
@@ -433,6 +436,20 @@
raise ValueError("netloc '" + netloc + "' contains invalid " +
"characters under NFKC normalization")
+# Valid bracketed hosts are defined in
+# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/
+def _check_bracketed_host(hostname):
+ if hostname.startswith('v'):
+ if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname):
+ raise ValueError(f"IPvFuture address is invalid")
+ else:
+ ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4
+ if isinstance(ip, ipaddress.IPv4Address):
+ raise ValueError(f"An IPv4 address cannot be in brackets")
+
+# typed=True avoids BytesWarnings being emitted during cache key
+# comparison since this API supports both bytes and str input.
+@functools.lru_cache(typed=True)
def urlsplit(url, scheme='', allow_fragments=True):
"""Parse a URL into 5 components:
<scheme>://<netloc>/<path>?<query>#<fragment>
@@ -455,39 +472,38 @@
"""
url, scheme, _coerce_result = _coerce_args(url, scheme)
+ # Only lstrip url as some applications rely on preserving trailing space.
+ # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both)
+ url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE)
+ scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE)
for b in _UNSAFE_URL_BYTES_TO_REMOVE:
url = url.replace(b, "")
scheme = scheme.replace(b, "")
allow_fragments = bool(allow_fragments)
- key = url, scheme, allow_fragments, type(url), type(scheme)
- cached = _parse_cache.get(key, None)
- if cached:
- return _coerce_result(cached)
- if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth
- clear_cache()
netloc = query = fragment = ''
i = url.find(':')
- if i > 0:
+ if i > 0 and url[0].isascii() and url[0].isalpha():
for c in url[:i]:
if c not in scheme_chars:
break
else:
scheme, url = url[:i].lower(), url[i+1:]
-
if url[:2] == '//':
netloc, url = _splitnetloc(url, 2)
if (('[' in netloc and ']' not in netloc) or
(']' in netloc and '[' not in netloc)):
raise ValueError("Invalid IPv6 URL")
+ if '[' in netloc and ']' in netloc:
+ bracketed_host = netloc.partition('[')[2].partition(']')[0]
+ _check_bracketed_host(bracketed_host)
if allow_fragments and '#' in url:
url, fragment = url.split('#', 1)
if '?' in url:
url, query = url.split('?', 1)
_checknetloc(netloc)
v = SplitResult(scheme, netloc, url, query, fragment)
- _parse_cache[key] = v
return _coerce_result(v)
def urlunparse(components):
@@ -749,12 +765,13 @@
# is less than max_num_fields. This prevents a memory exhaustion DOS
# attack via post bodies with many fields.
if max_num_fields is not None:
- num_fields = 1 + qs.count(separator)
+ num_fields = 1 + qs.count(separator) if qs else 0
if max_num_fields < num_fields:
raise ValueError('Max number of fields exceeded')
r = []
- for name_value in qs.split(separator):
+ query_args = qs.split(separator) if qs else []
+ for name_value in query_args:
if not name_value and not strict_parsing:
continue
nv = name_value.split('=', 1)
@@ -790,23 +807,30 @@
b'0123456789'
b'_.-~')
_ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE)
-_safe_quoters = {}
-class Quoter(collections.defaultdict):
- """A mapping from bytes (in range(0,256)) to strings.
+def __getattr__(name):
+ if name == 'Quoter':
+ warnings.warn('Deprecated in 3.11. '
+ 'urllib.parse.Quoter will be removed in Python 3.14. '
+ 'It was not intended to be a public API.',
+ DeprecationWarning, stacklevel=2)
+ return _Quoter
+ raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
+
+class _Quoter(dict):
+ """A mapping from bytes numbers (in range(0,256)) to strings.
String values are percent-encoded byte values, unless the key < 128, and
- in the "safe" set (either the specified safe set, or default set).
+ in either of the specified safe set, or the always safe set.
"""
- # Keeps a cache internally, using defaultdict, for efficiency (lookups
+ # Keeps a cache internally, via __missing__, for efficiency (lookups
# of cached keys don't call Python code at all).
def __init__(self, safe):
"""safe: bytes object."""
self.safe = _ALWAYS_SAFE.union(safe)
def __repr__(self):
- # Without this, will just display as a defaultdict
- return "<%s %r>" % (self.__class__.__name__, dict(self))
+ return f"<Quoter {dict(self)!r}>"
def __missing__(self, b):
# Handle a cache miss. Store quoted string in cache and return.
@@ -885,6 +909,11 @@
string = quote(string, safe + space, encoding, errors)
return string.replace(' ', '+')
+# Expectation: A typical program is unlikely to create more than 5 of these.
+@functools.lru_cache
+def _byte_quoter_factory(safe):
+ return _Quoter(safe).__getitem__
+
def quote_from_bytes(bs, safe='/'):
"""Like quote(), but accepts a bytes object rather than a str, and does
not perform string-to-bytes encoding. It always returns an ASCII string.
@@ -898,13 +927,11 @@
# Normalize 'safe' by converting to bytes and removing non-ASCII chars
safe = safe.encode('ascii', 'ignore')
else:
+ # List comprehensions are faster than generator expressions.
safe = bytes([c for c in safe if c < 128])
if not bs.rstrip(_ALWAYS_SAFE_BYTES + safe):
return bs.decode()
- try:
- quoter = _safe_quoters[safe]
- except KeyError:
- _safe_quoters[safe] = quoter = Quoter(safe).__getitem__
+ quoter = _byte_quoter_factory(safe)
return ''.join([quoter(char) for char in bs])
def urlencode(query, doseq=False, safe='', encoding=None, errors=None,
@@ -938,10 +965,9 @@
# but that's a minor nit. Since the original implementation
# allowed empty dicts that type of behavior probably should be
# preserved for consistency
- except TypeError:
- ty, va, tb = sys.exc_info()
+ except TypeError as err:
raise TypeError("not a valid non-string sequence "
- "or mapping object").with_traceback(tb)
+ "or mapping object") from err
l = []
if not doseq:
diff --git a/common/py3-stdlib/urllib/request.py b/common/py3-stdlib/urllib/request.py
index 76fdd71..24911bb 100644
--- a/common/py3-stdlib/urllib/request.py
+++ b/common/py3-stdlib/urllib/request.py
@@ -11,8 +11,8 @@
Handlers needed to open the requested URL. For example, the
HTTPHandler performs HTTP GET and POST requests and deals with
non-error returns. The HTTPRedirectHandler automatically deals with
-HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
-deals with digest authentication.
+HTTP 301, 302, 303, 307, and 308 redirect errors, and the
+HTTPDigestAuthHandler deals with digest authentication.
urlopen(url, data=None) -- Basic usage is the same as original
urllib. pass the url and optionally data to post to an HTTP URL, and
@@ -661,7 +661,7 @@
but another Handler might.
"""
m = req.get_method()
- if (not (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+ if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
or code in (301, 302, 303) and m == "POST")):
raise HTTPError(req.full_url, code, msg, headers, fp)
@@ -748,7 +748,7 @@
return self.parent.open(new, timeout=req.timeout)
- http_error_301 = http_error_303 = http_error_307 = http_error_302
+ http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302
inf_msg = "The HTTP server returned a redirect error that would " \
"lead to an infinite loop.\n" \
@@ -1790,7 +1790,7 @@
except (HTTPError, URLError):
raise
except OSError as msg:
- raise OSError('socket error', msg).with_traceback(sys.exc_info()[2])
+ raise OSError('socket error', msg) from msg
def open_unknown(self, fullurl, data=None):
"""Overridable interface to open unknown URL type."""
@@ -2092,7 +2092,7 @@
headers = email.message_from_string(headers)
return addinfourl(fp, headers, "ftp:" + url)
except ftperrors() as exp:
- raise URLError('ftp error %r' % exp).with_traceback(sys.exc_info()[2])
+ raise URLError(f'ftp error: {exp}') from exp
def open_data(self, url, data=None):
"""Use "data" URL."""
@@ -2210,6 +2210,13 @@
else:
return self.http_error_default(url, fp, errcode, errmsg, headers)
+ def http_error_308(self, url, fp, errcode, errmsg, headers, data=None):
+ """Error 308 -- relocated, but turn POST into error."""
+ if data is None:
+ return self.http_error_301(url, fp, errcode, errmsg, headers, data)
+ else:
+ return self.http_error_default(url, fp, errcode, errmsg, headers)
+
def http_error_401(self, url, fp, errcode, errmsg, headers, data=None,
retry=False):
"""Error 401 -- authentication required.
@@ -2435,8 +2442,7 @@
conn, retrlen = self.ftp.ntransfercmd(cmd)
except ftplib.error_perm as reason:
if str(reason)[:3] != '550':
- raise URLError('ftp error: %r' % reason).with_traceback(
- sys.exc_info()[2])
+ raise URLError(f'ftp error: {reason}') from reason
if not conn:
# Set transfer mode to ASCII!
self.ftp.voidcmd('TYPE A')
@@ -2463,7 +2469,13 @@
return (ftpobj, retrlen)
def endtransfer(self):
+ if not self.busy:
+ return
self.busy = 0
+ try:
+ self.ftp.voidresp()
+ except ftperrors():
+ pass
def close(self):
self.keepalive = False
@@ -2491,34 +2503,28 @@
this seems to be the standard convention. If you need a
different way, you can pass a proxies dictionary to the
[Fancy]URLopener constructor.
+
"""
+ proxies = {}
# in order to prefer lowercase variables, process environment in
# two passes: first matches any, second pass matches lowercase only
-
- # select only environment variables which end in (after making lowercase) _proxy
- proxies = {}
- environment = []
- for name in os.environ.keys():
- # fast screen underscore position before more expensive case-folding
- if len(name) > 5 and name[-6] == "_" and name[-5:].lower() == "proxy":
- value = os.environ[name]
- proxy_name = name[:-6].lower()
- environment.append((name, value, proxy_name))
- if value:
- proxies[proxy_name] = value
+ for name, value in os.environ.items():
+ name = name.lower()
+ if value and name[-6:] == '_proxy':
+ proxies[name[:-6]] = value
# CVE-2016-1000110 - If we are running as CGI script, forget HTTP_PROXY
# (non-all-lowercase) as it may be set from the web server by a "Proxy:"
# header from the client
# If "proxy" is lowercase, it will still be used thanks to the next block
if 'REQUEST_METHOD' in os.environ:
proxies.pop('http', None)
- for name, value, proxy_name in environment:
- # not case-folded, checking here for lower-case env vars only
+ for name, value in os.environ.items():
if name[-6:] == '_proxy':
+ name = name.lower()
if value:
- proxies[proxy_name] = value
+ proxies[name[:-6]] = value
else:
- proxies.pop(proxy_name, None)
+ proxies.pop(name[:-6], None)
return proxies
def proxy_bypass_environment(host, proxies=None):
diff --git a/common/py3-stdlib/uu.py b/common/py3-stdlib/uu.py
old mode 100755
new mode 100644
index 9f1f37f..26bb59a
--- a/common/py3-stdlib/uu.py
+++ b/common/py3-stdlib/uu.py
@@ -33,6 +33,9 @@
import binascii
import os
import sys
+import warnings
+
+warnings._deprecated(__name__, remove=(3, 13))
__all__ = ["Error", "encode", "decode"]
@@ -130,7 +133,14 @@
# If the filename isn't ASCII, what's up with that?!?
out_file = hdrfields[2].rstrip(b' \t\r\n\f').decode("ascii")
if os.path.exists(out_file):
- raise Error('Cannot overwrite existing file: %s' % out_file)
+ raise Error(f'Cannot overwrite existing file: {out_file}')
+ if (out_file.startswith(os.sep) or
+ f'..{os.sep}' in out_file or (
+ os.altsep and
+ (out_file.startswith(os.altsep) or
+ f'..{os.altsep}' in out_file))
+ ):
+ raise Error(f'Refusing to write to {out_file} due to directory traversal')
if mode is None:
mode = int(hdrfields[1], 8)
#
diff --git a/common/py3-stdlib/uuid.py b/common/py3-stdlib/uuid.py
index fe9f87b..e863b63 100644
--- a/common/py3-stdlib/uuid.py
+++ b/common/py3-stdlib/uuid.py
@@ -47,7 +47,7 @@
import os
import sys
-from enum import Enum
+from enum import Enum, _simple_enum
__author__ = 'Ka-Ping Yee <ping@zesty.ca>'
@@ -75,7 +75,8 @@
bytes_ = bytes # The built-in bytes type
-class SafeUUID(Enum):
+@_simple_enum(Enum)
+class SafeUUID:
safe = 0
unsafe = -1
unknown = None
@@ -185,7 +186,7 @@
if len(bytes) != 16:
raise ValueError('bytes is not a 16-char string')
assert isinstance(bytes, bytes_), repr(bytes)
- int = int_.from_bytes(bytes, byteorder='big')
+ int = int_.from_bytes(bytes) # big endian
if fields is not None:
if len(fields) != 6:
raise ValueError('fields is not a 6-tuple')
@@ -283,7 +284,7 @@
@property
def bytes(self):
- return self.int.to_bytes(16, 'big')
+ return self.int.to_bytes(16) # big endian
@property
def bytes_le(self):
@@ -528,6 +529,8 @@
def _arp_getnode():
"""Get the hardware address on Unix by running arp."""
import os, socket
+ if not hasattr(socket, "gethostbyname"):
+ return None
try:
ip_addr = socket.gethostbyname(socket.gethostname())
except OSError:
diff --git a/common/py3-stdlib/warnings.py b/common/py3-stdlib/warnings.py
index 691ccdd..7d8c440 100644
--- a/common/py3-stdlib/warnings.py
+++ b/common/py3-stdlib/warnings.py
@@ -432,9 +432,13 @@
named 'warnings' and imported under that name. This argument is only useful
when testing the warnings module itself.
+ If the 'action' argument is not None, the remaining arguments are passed
+ to warnings.simplefilter() as if it were called immediately on entering the
+ context.
"""
- def __init__(self, *, record=False, module=None):
+ def __init__(self, *, record=False, module=None,
+ action=None, category=Warning, lineno=0, append=False):
"""Specify whether to record warnings and if an alternative module
should be used other than sys.modules['warnings'].
@@ -445,6 +449,10 @@
self._record = record
self._module = sys.modules['warnings'] if module is None else module
self._entered = False
+ if action is None:
+ self._filter = None
+ else:
+ self._filter = (action, category, lineno, append)
def __repr__(self):
args = []
@@ -464,6 +472,8 @@
self._module._filters_mutated()
self._showwarning = self._module.showwarning
self._showwarnmsg_impl = self._module._showwarnmsg_impl
+ if self._filter is not None:
+ simplefilter(*self._filter)
if self._record:
log = []
self._module._showwarnmsg_impl = log.append
@@ -483,6 +493,27 @@
self._module._showwarnmsg_impl = self._showwarnmsg_impl
+_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}"
+
+def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info):
+ """Warn that *name* is deprecated or should be removed.
+
+ RuntimeError is raised if *remove* specifies a major/minor tuple older than
+ the current Python version or the same version but past the alpha.
+
+ The *message* argument is formatted with *name* and *remove* as a Python
+ version (e.g. "3.11").
+
+ """
+ remove_formatted = f"{remove[0]}.{remove[1]}"
+ if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"):
+ msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha"
+ raise RuntimeError(msg)
+ else:
+ msg = message.format(name=name, remove=remove_formatted)
+ warn(msg, DeprecationWarning, stacklevel=3)
+
+
# Private utility function called by _PyErr_WarnUnawaitedCoroutine
def _warn_unawaited_coroutine(coro):
msg_lines = [
diff --git a/common/py3-stdlib/wave.py b/common/py3-stdlib/wave.py
index b707119..bcacbac 100644
--- a/common/py3-stdlib/wave.py
+++ b/common/py3-stdlib/wave.py
@@ -71,9 +71,7 @@
is destroyed.
"""
-from chunk import Chunk
from collections import namedtuple
-import audioop
import builtins
import struct
import sys
@@ -91,6 +89,128 @@
_wave_params = namedtuple('_wave_params',
'nchannels sampwidth framerate nframes comptype compname')
+
+def _byteswap(data, width):
+ swapped_data = bytearray(len(data))
+
+ for i in range(0, len(data), width):
+ for j in range(width):
+ swapped_data[i + width - 1 - j] = data[i + j]
+
+ return bytes(swapped_data)
+
+
+class _Chunk:
+ def __init__(self, file, align=True, bigendian=True, inclheader=False):
+ self.closed = False
+ self.align = align # whether to align to word (2-byte) boundaries
+ if bigendian:
+ strflag = '>'
+ else:
+ strflag = '<'
+ self.file = file
+ self.chunkname = file.read(4)
+ if len(self.chunkname) < 4:
+ raise EOFError
+ try:
+ self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
+ except struct.error:
+ raise EOFError from None
+ if inclheader:
+ self.chunksize = self.chunksize - 8 # subtract header
+ self.size_read = 0
+ try:
+ self.offset = self.file.tell()
+ except (AttributeError, OSError):
+ self.seekable = False
+ else:
+ self.seekable = True
+
+ def getname(self):
+ """Return the name (ID) of the current chunk."""
+ return self.chunkname
+
+ def close(self):
+ if not self.closed:
+ try:
+ self.skip()
+ finally:
+ self.closed = True
+
+ def seek(self, pos, whence=0):
+ """Seek to specified position into the chunk.
+ Default position is 0 (start of chunk).
+ If the file is not seekable, this will result in an error.
+ """
+
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ if not self.seekable:
+ raise OSError("cannot seek")
+ if whence == 1:
+ pos = pos + self.size_read
+ elif whence == 2:
+ pos = pos + self.chunksize
+ if pos < 0 or pos > self.chunksize:
+ raise RuntimeError
+ self.file.seek(self.offset + pos, 0)
+ self.size_read = pos
+
+ def tell(self):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ return self.size_read
+
+ def read(self, size=-1):
+ """Read at most size bytes from the chunk.
+ If size is omitted or negative, read until the end
+ of the chunk.
+ """
+
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ if self.size_read >= self.chunksize:
+ return b''
+ if size < 0:
+ size = self.chunksize - self.size_read
+ if size > self.chunksize - self.size_read:
+ size = self.chunksize - self.size_read
+ data = self.file.read(size)
+ self.size_read = self.size_read + len(data)
+ if self.size_read == self.chunksize and \
+ self.align and \
+ (self.chunksize & 1):
+ dummy = self.file.read(1)
+ self.size_read = self.size_read + len(dummy)
+ return data
+
+ def skip(self):
+ """Skip the rest of the chunk.
+ If you are not interested in the contents of the chunk,
+ this method should be called so that the file points to
+ the start of the next chunk.
+ """
+
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+ if self.seekable:
+ try:
+ n = self.chunksize - self.size_read
+ # maybe fix alignment
+ if self.align and (self.chunksize & 1):
+ n = n + 1
+ self.file.seek(n, 1)
+ self.size_read = self.size_read + n
+ return
+ except OSError:
+ pass
+ while self.size_read < self.chunksize:
+ n = min(8192, self.chunksize - self.size_read)
+ dummy = self.read(n)
+ if not dummy:
+ raise EOFError
+
+
class Wave_read:
"""Variables used in this class:
@@ -125,7 +245,7 @@
def initfp(self, file):
self._convert = None
self._soundpos = 0
- self._file = Chunk(file, bigendian = 0)
+ self._file = _Chunk(file, bigendian = 0)
if self._file.getname() != b'RIFF':
raise Error('file does not start with RIFF id')
if self._file.read(4) != b'WAVE':
@@ -135,7 +255,7 @@
while 1:
self._data_seek_needed = 1
try:
- chunk = Chunk(self._file, bigendian = 0)
+ chunk = _Chunk(self._file, bigendian = 0)
except EOFError:
break
chunkname = chunk.getname()
@@ -241,7 +361,7 @@
return b''
data = self._data_chunk.read(nframes * self._framesize)
if self._sampwidth != 1 and sys.byteorder == 'big':
- data = audioop.byteswap(data, self._sampwidth)
+ data = _byteswap(data, self._sampwidth)
if self._convert and data:
data = self._convert(data)
self._soundpos = self._soundpos + len(data) // (self._nchannels * self._sampwidth)
@@ -272,6 +392,7 @@
self._comptype = 'NONE'
self._compname = 'not compressed'
+
class Wave_write:
"""Variables used in this class:
@@ -428,7 +549,7 @@
if self._convert:
data = self._convert(data)
if self._sampwidth != 1 and sys.byteorder == 'big':
- data = audioop.byteswap(data, self._sampwidth)
+ data = _byteswap(data, self._sampwidth)
self._file.write(data)
self._datawritten += len(data)
self._nframeswritten = self._nframeswritten + nframes
@@ -499,6 +620,7 @@
self._file.seek(curpos, 0)
self._datalength = self._datawritten
+
def open(f, mode=None):
if mode is None:
if hasattr(f, 'mode'):
diff --git a/common/py3-stdlib/weakref.py b/common/py3-stdlib/weakref.py
index 994ea8a..25b7092 100644
--- a/common/py3-stdlib/weakref.py
+++ b/common/py3-stdlib/weakref.py
@@ -2,7 +2,7 @@
This module is an implementation of PEP 205:
-https://www.python.org/dev/peps/pep-0205/
+https://peps.python.org/pep-0205/
"""
# Naming convention: Variables named "wr" are weak reference objects;
@@ -33,7 +33,6 @@
"WeakSet", "WeakMethod", "finalize"]
-_collections_abc.Set.register(WeakSet)
_collections_abc.MutableSet.register(WeakSet)
class WeakMethod(ref):
diff --git a/common/py3-stdlib/webbrowser.py b/common/py3-stdlib/webbrowser.py
index ec3cece..44974d4 100755
--- a/common/py3-stdlib/webbrowser.py
+++ b/common/py3-stdlib/webbrowser.py
@@ -8,6 +8,7 @@
import sys
import subprocess
import threading
+import warnings
__all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"]
@@ -461,14 +462,15 @@
if shutil.which("xdg-open"):
register("xdg-open", None, BackgroundBrowser("xdg-open"))
- # The default GNOME3 browser
+ # Opens an appropriate browser for the URL scheme according to
+ # freedesktop.org settings (GNOME, KDE, XFCE, etc.)
+ if shutil.which("gio"):
+ register("gio", None, BackgroundBrowser(["gio", "open", "--", "%s"]))
+
+ # Equivalent of gio open before 2015
if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"):
register("gvfs-open", None, BackgroundBrowser("gvfs-open"))
- # The default GNOME browser
- if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gnome-open"):
- register("gnome-open", None, BackgroundBrowser("gnome-open"))
-
# The default KDE browser
if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"):
register("kfmclient", Konqueror, Konqueror("kfmclient"))
@@ -629,6 +631,8 @@
Internet System Preferences panel, will be used.
"""
def __init__(self, name):
+ warnings.warn(f'{self.__class__.__name__} is deprecated in 3.11'
+ ' use MacOSXOSAScript instead.', DeprecationWarning, stacklevel=2)
self.name = name
def open(self, url, new=0, autoraise=True):
@@ -666,19 +670,33 @@
return not rc
class MacOSXOSAScript(BaseBrowser):
- def __init__(self, name):
- self._name = name
+ def __init__(self, name='default'):
+ super().__init__(name)
+
+ @property
+ def _name(self):
+ warnings.warn(f'{self.__class__.__name__}._name is deprecated in 3.11'
+ f' use {self.__class__.__name__}.name instead.',
+ DeprecationWarning, stacklevel=2)
+ return self.name
+
+ @_name.setter
+ def _name(self, val):
+ warnings.warn(f'{self.__class__.__name__}._name is deprecated in 3.11'
+ f' use {self.__class__.__name__}.name instead.',
+ DeprecationWarning, stacklevel=2)
+ self.name = val
def open(self, url, new=0, autoraise=True):
- if self._name == 'default':
+ if self.name == 'default':
script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser
else:
- script = '''
+ script = f'''
tell application "%s"
activate
open location "%s"
end
- '''%(self._name, url.replace('"', '%22'))
+ '''%(self.name, url.replace('"', '%22'))
osapipe = os.popen("osascript", "w")
if osapipe is None:
diff --git a/common/py3-stdlib/wsgiref/__init__.py b/common/py3-stdlib/wsgiref/__init__.py
index 1efbba0..59ee48f 100644
--- a/common/py3-stdlib/wsgiref/__init__.py
+++ b/common/py3-stdlib/wsgiref/__init__.py
@@ -13,6 +13,8 @@
* validate -- validation wrapper that sits between an app and a server
to detect errors in either
+* types -- collection of WSGI-related types for static type checking
+
To-Do:
* cgi_gateway -- Run WSGI apps under CGI (pending a deployment standard)
diff --git a/common/py3-stdlib/wsgiref/handlers.py b/common/py3-stdlib/wsgiref/handlers.py
index 31360e5..6623b70 100644
--- a/common/py3-stdlib/wsgiref/handlers.py
+++ b/common/py3-stdlib/wsgiref/handlers.py
@@ -228,8 +228,7 @@
if exc_info:
try:
if self.headers_sent:
- # Re-raise original exception if headers sent
- raise exc_info[0](exc_info[1]).with_traceback(exc_info[2])
+ raise
finally:
exc_info = None # avoid dangling circular ref
elif self.headers is not None:
diff --git a/common/py3-stdlib/wsgiref/types.py b/common/py3-stdlib/wsgiref/types.py
new file mode 100644
index 0000000..ef0aead
--- /dev/null
+++ b/common/py3-stdlib/wsgiref/types.py
@@ -0,0 +1,54 @@
+"""WSGI-related types for static type checking"""
+
+from collections.abc import Callable, Iterable, Iterator
+from types import TracebackType
+from typing import Any, Protocol, TypeAlias
+
+__all__ = [
+ "StartResponse",
+ "WSGIEnvironment",
+ "WSGIApplication",
+ "InputStream",
+ "ErrorStream",
+ "FileWrapper",
+]
+
+_ExcInfo: TypeAlias = tuple[type[BaseException], BaseException, TracebackType]
+_OptExcInfo: TypeAlias = _ExcInfo | tuple[None, None, None]
+
+class StartResponse(Protocol):
+ """start_response() callable as defined in PEP 3333"""
+ def __call__(
+ self,
+ status: str,
+ headers: list[tuple[str, str]],
+ exc_info: _OptExcInfo | None = ...,
+ /,
+ ) -> Callable[[bytes], object]: ...
+
+WSGIEnvironment: TypeAlias = dict[str, Any]
+WSGIApplication: TypeAlias = Callable[[WSGIEnvironment, StartResponse],
+ Iterable[bytes]]
+
+class InputStream(Protocol):
+ """WSGI input stream as defined in PEP 3333"""
+ def read(self, size: int = ..., /) -> bytes: ...
+ def readline(self, size: int = ..., /) -> bytes: ...
+ def readlines(self, hint: int = ..., /) -> list[bytes]: ...
+ def __iter__(self) -> Iterator[bytes]: ...
+
+class ErrorStream(Protocol):
+ """WSGI error stream as defined in PEP 3333"""
+ def flush(self) -> object: ...
+ def write(self, s: str, /) -> object: ...
+ def writelines(self, seq: list[str], /) -> object: ...
+
+class _Readable(Protocol):
+ def read(self, size: int = ..., /) -> bytes: ...
+ # Optional: def close(self) -> object: ...
+
+class FileWrapper(Protocol):
+ """WSGI file wrapper as defined in PEP 3333"""
+ def __call__(
+ self, file: _Readable, block_size: int = ..., /,
+ ) -> Iterable[bytes]: ...
diff --git a/common/py3-stdlib/wsgiref/util.py b/common/py3-stdlib/wsgiref/util.py
index cac52eb..cbbe094 100644
--- a/common/py3-stdlib/wsgiref/util.py
+++ b/common/py3-stdlib/wsgiref/util.py
@@ -17,19 +17,6 @@
if hasattr(filelike,'close'):
self.close = filelike.close
- def __getitem__(self,key):
- import warnings
- warnings.warn(
- "FileWrapper's __getitem__ method ignores 'key' parameter. "
- "Use iterator protocol instead.",
- DeprecationWarning,
- stacklevel=2
- )
- data = self.filelike.read(self.blksize)
- if data:
- return data
- raise IndexError
-
def __iter__(self):
return self
diff --git a/common/py3-stdlib/xdrlib.py b/common/py3-stdlib/xdrlib.py
index d6e1aeb..b56ffa5 100644
--- a/common/py3-stdlib/xdrlib.py
+++ b/common/py3-stdlib/xdrlib.py
@@ -7,6 +7,9 @@
import struct
from io import BytesIO
from functools import wraps
+import warnings
+
+warnings._deprecated(__name__, remove=(3, 13))
__all__ = ["Error", "Packer", "Unpacker", "ConversionError"]
diff --git a/common/py3-stdlib/xml/dom/pulldom.py b/common/py3-stdlib/xml/dom/pulldom.py
index 96a8d59..913141c 100644
--- a/common/py3-stdlib/xml/dom/pulldom.py
+++ b/common/py3-stdlib/xml/dom/pulldom.py
@@ -216,19 +216,6 @@
self.parser.setFeature(xml.sax.handler.feature_namespaces, 1)
self.parser.setContentHandler(self.pulldom)
- def __getitem__(self, pos):
- import warnings
- warnings.warn(
- "DOMEventStream's __getitem__ method ignores 'pos' parameter. "
- "Use iterator protocol instead.",
- DeprecationWarning,
- stacklevel=2
- )
- rc = self.getEvent()
- if rc:
- return rc
- raise IndexError
-
def __next__(self):
rc = self.getEvent()
if rc:
diff --git a/common/py3-stdlib/xml/etree/ElementPath.py b/common/py3-stdlib/xml/etree/ElementPath.py
index cd3c354..dc6bd28 100644
--- a/common/py3-stdlib/xml/etree/ElementPath.py
+++ b/common/py3-stdlib/xml/etree/ElementPath.py
@@ -416,6 +416,8 @@
def findtext(elem, path, default=None, namespaces=None):
try:
elem = next(iterfind(elem, path, namespaces))
- return elem.text or ""
+ if elem.text is None:
+ return ""
+ return elem.text
except StopIteration:
return default
diff --git a/common/py3-stdlib/xml/etree/ElementTree.py b/common/py3-stdlib/xml/etree/ElementTree.py
index 2503d9e..1dc8035 100644
--- a/common/py3-stdlib/xml/etree/ElementTree.py
+++ b/common/py3-stdlib/xml/etree/ElementTree.py
@@ -911,13 +911,9 @@
if elem.tail:
write(_escape_cdata(elem.tail))
-HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr",
- "img", "input", "isindex", "link", "meta", "param")
-
-try:
- HTML_EMPTY = set(HTML_EMPTY)
-except NameError:
- pass
+HTML_EMPTY = {"area", "base", "basefont", "br", "col", "embed", "frame", "hr",
+ "img", "input", "isindex", "link", "meta", "param", "source",
+ "track", "wbr"}
def _serialize_html(write, elem, qnames, namespaces, **kwargs):
tag = elem.tag
diff --git a/common/py3-stdlib/xmlrpc/server.py b/common/py3-stdlib/xmlrpc/server.py
index 69a260f..0c4b558 100644
--- a/common/py3-stdlib/xmlrpc/server.py
+++ b/common/py3-stdlib/xmlrpc/server.py
@@ -268,17 +268,11 @@
except Fault as fault:
response = dumps(fault, allow_none=self.allow_none,
encoding=self.encoding)
- except:
- # report exception back to server
- exc_type, exc_value, exc_tb = sys.exc_info()
- try:
- response = dumps(
- Fault(1, "%s:%s" % (exc_type, exc_value)),
- encoding=self.encoding, allow_none=self.allow_none,
- )
- finally:
- # Break reference cycle
- exc_type = exc_value = exc_tb = None
+ except BaseException as exc:
+ response = dumps(
+ Fault(1, "%s:%s" % (type(exc), exc)),
+ encoding=self.encoding, allow_none=self.allow_none,
+ )
return response.encode(self.encoding, 'xmlcharrefreplace')
@@ -368,16 +362,11 @@
{'faultCode' : fault.faultCode,
'faultString' : fault.faultString}
)
- except:
- exc_type, exc_value, exc_tb = sys.exc_info()
- try:
- results.append(
- {'faultCode' : 1,
- 'faultString' : "%s:%s" % (exc_type, exc_value)}
- )
- finally:
- # Break reference cycle
- exc_type = exc_value = exc_tb = None
+ except BaseException as exc:
+ results.append(
+ {'faultCode' : 1,
+ 'faultString' : "%s:%s" % (type(exc), exc)}
+ )
return results
def _dispatch(self, method, params):
@@ -440,7 +429,7 @@
# Class attribute listing the accessible path components;
# paths not on this list will result in a 404 error.
- rpc_paths = ('/', '/RPC2')
+ rpc_paths = ('/', '/RPC2', '/pydoc.css')
#if not None, encode responses larger than this, if possible
encode_threshold = 1400 #a common MTU
@@ -634,19 +623,14 @@
try:
response = self.dispatchers[path]._marshaled_dispatch(
data, dispatch_method, path)
- except:
+ except BaseException as exc:
# report low level exception back to server
# (each dispatcher should have handled their own
# exceptions)
- exc_type, exc_value = sys.exc_info()[:2]
- try:
- response = dumps(
- Fault(1, "%s:%s" % (exc_type, exc_value)),
- encoding=self.encoding, allow_none=self.allow_none)
- response = response.encode(self.encoding, 'xmlcharrefreplace')
- finally:
- # Break reference cycle
- exc_type = exc_value = None
+ response = dumps(
+ Fault(1, "%s:%s" % (type(exc), exc)),
+ encoding=self.encoding, allow_none=self.allow_none)
+ response = response.encode(self.encoding, 'xmlcharrefreplace')
return response
class CGIXMLRPCRequestHandler(SimpleXMLRPCDispatcher):
@@ -747,10 +731,10 @@
url = escape(all).replace('"', '"')
results.append('<a href="%s">%s</a>' % (url, url))
elif rfc:
- url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
+ url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif pep:
- url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep)
+ url = 'https://peps.python.org/pep-%04d/' % int(pep)
results.append('<a href="%s">%s</a>' % (url, escape(all)))
elif text[end:end+1] == '(':
results.append(self.namelink(name, methods, funcs, classes))
@@ -801,7 +785,7 @@
server_name = self.escape(server_name)
head = '<big><big><strong>%s</strong></big></big>' % server_name
- result = self.heading(head, '#ffffff', '#7799ee')
+ result = self.heading(head)
doc = self.markup(package_documentation, self.preformat, fdict)
doc = doc and '<tt>%s</tt>' % doc
@@ -812,10 +796,25 @@
for key, value in method_items:
contents.append(self.docroutine(value, key, funcs=fdict))
result = result + self.bigsection(
- 'Methods', '#ffffff', '#eeaa77', ''.join(contents))
+ 'Methods', 'functions', ''.join(contents))
return result
+
+ def page(self, title, contents):
+ """Format an HTML page."""
+ css_path = "/pydoc.css"
+ css_link = (
+ '<link rel="stylesheet" type="text/css" href="%s">' %
+ css_path)
+ return '''\
+<!DOCTYPE>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Python: %s</title>
+%s</head><body>%s</body></html>''' % (title, css_link, contents)
+
class XMLRPCDocGenerator:
"""Generates documentation for an XML-RPC server.
@@ -907,6 +906,12 @@
for documentation.
"""
+ def _get_css(self, url):
+ path_here = os.path.dirname(os.path.realpath(__file__))
+ css_path = os.path.join(path_here, "..", "pydoc_data", "_pydoc.css")
+ with open(css_path, mode="rb") as fp:
+ return fp.read()
+
def do_GET(self):
"""Handles the HTTP GET request.
@@ -918,9 +923,15 @@
self.report_404()
return
- response = self.server.generate_html_documentation().encode('utf-8')
+ if self.path.endswith('.css'):
+ content_type = 'text/css'
+ response = self._get_css(self.path)
+ else:
+ content_type = 'text/html'
+ response = self.server.generate_html_documentation().encode('utf-8')
+
self.send_response(200)
- self.send_header("Content-type", "text/html")
+ self.send_header('Content-Type', '%s; charset=UTF-8' % content_type)
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
diff --git a/common/py3-stdlib/zipfile.py b/common/py3-stdlib/zipfile.py
index 42e11d7..1dec6a8 100644
--- a/common/py3-stdlib/zipfile.py
+++ b/common/py3-stdlib/zipfile.py
@@ -123,6 +123,32 @@
_CD_EXTERNAL_FILE_ATTRIBUTES = 17
_CD_LOCAL_HEADER_OFFSET = 18
+# General purpose bit flags
+# Zip Appnote: 4.4.4 general purpose bit flag: (2 bytes)
+_MASK_ENCRYPTED = 1 << 0
+# Bits 1 and 2 have different meanings depending on the compression used.
+_MASK_COMPRESS_OPTION_1 = 1 << 1
+# _MASK_COMPRESS_OPTION_2 = 1 << 2
+# _MASK_USE_DATA_DESCRIPTOR: If set, crc-32, compressed size and uncompressed
+# size are zero in the local header and the real values are written in the data
+# descriptor immediately following the compressed data.
+_MASK_USE_DATA_DESCRIPTOR = 1 << 3
+# Bit 4: Reserved for use with compression method 8, for enhanced deflating.
+# _MASK_RESERVED_BIT_4 = 1 << 4
+_MASK_COMPRESSED_PATCH = 1 << 5
+_MASK_STRONG_ENCRYPTION = 1 << 6
+# _MASK_UNUSED_BIT_7 = 1 << 7
+# _MASK_UNUSED_BIT_8 = 1 << 8
+# _MASK_UNUSED_BIT_9 = 1 << 9
+# _MASK_UNUSED_BIT_10 = 1 << 10
+_MASK_UTF_FILENAME = 1 << 11
+# Bit 12: Reserved by PKWARE for enhanced compression.
+# _MASK_RESERVED_BIT_12 = 1 << 12
+# _MASK_ENCRYPTED_CENTRAL_DIR = 1 << 13
+# Bit 14, 15: Reserved by PKWARE
+# _MASK_RESERVED_BIT_14 = 1 << 14
+# _MASK_RESERVED_BIT_15 = 1 << 15
+
# The "local file header" structure, magic number, size, and indices
# (section V.A in the format document)
structFileHeader = "<4s2B4HL2L2H"
@@ -409,11 +435,16 @@
return ''.join(result)
def FileHeader(self, zip64=None):
- """Return the per-file header as a bytes object."""
+ """Return the per-file header as a bytes object.
+
+ When the optional zip64 arg is None rather than a bool, we will
+ decide based upon the file_size and compress_size, if known,
+ False otherwise.
+ """
dt = self.date_time
dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2]
dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2)
- if self.flag_bits & 0x08:
+ if self.flag_bits & _MASK_USE_DATA_DESCRIPTOR:
# Set these to zero because we write them after the file data
CRC = compress_size = file_size = 0
else:
@@ -425,16 +456,13 @@
min_version = 0
if zip64 is None:
+ # We always explicitly pass zip64 within this module.... This
+ # remains for anyone using ZipInfo.FileHeader as a public API.
zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT
if zip64:
fmt = '<HHQQ'
extra = extra + struct.pack(fmt,
1, struct.calcsize(fmt)-4, file_size, compress_size)
- if file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT:
- if not zip64:
- raise LargeZipFile("Filesize would require ZIP64 extensions")
- # File is larger than what fits into a 4 byte integer,
- # fall back to the ZIP64 extension
file_size = 0xffffffff
compress_size = 0xffffffff
min_version = ZIP64_VERSION
@@ -458,7 +486,7 @@
try:
return self.filename.encode('ascii'), self.flag_bits
except UnicodeEncodeError:
- return self.filename.encode('utf-8'), self.flag_bits | 0x800
+ return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME
def _decodeExtra(self):
# Try to decode the extra field.
@@ -829,7 +857,7 @@
self._decrypter = None
if pwd:
- if zipinfo.flag_bits & 0x8:
+ if zipinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR:
# compare against the file type from extended local headers
check_byte = (zipinfo._raw_time >> 8) & 0xff
else:
@@ -1157,21 +1185,20 @@
self._zinfo.CRC = self._crc
self._zinfo.file_size = self._file_size
+ if not self._zip64:
+ if self._file_size > ZIP64_LIMIT:
+ raise RuntimeError("File size too large, try using force_zip64")
+ if self._compress_size > ZIP64_LIMIT:
+ raise RuntimeError("Compressed size too large, try using force_zip64")
+
# Write updated header info
- if self._zinfo.flag_bits & 0x08:
+ if self._zinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR:
# Write CRC and file sizes after the file data
fmt = '<LLQQ' if self._zip64 else '<LLLL'
self._fileobj.write(struct.pack(fmt, _DD_SIGNATURE, self._zinfo.CRC,
self._zinfo.compress_size, self._zinfo.file_size))
self._zipfile.start_dir = self._fileobj.tell()
else:
- if not self._zip64:
- if self._file_size > ZIP64_LIMIT:
- raise RuntimeError(
- 'File size unexpectedly exceeded ZIP64 limit')
- if self._compress_size > ZIP64_LIMIT:
- raise RuntimeError(
- 'Compressed size unexpectedly exceeded ZIP64 limit')
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
@@ -1216,7 +1243,7 @@
_windows_illegal_name_trans_table = None
def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True,
- compresslevel=None, *, strict_timestamps=True):
+ compresslevel=None, *, strict_timestamps=True, metadata_encoding=None):
"""Open the ZIP file with mode read 'r', write 'w', exclusive create 'x',
or append 'a'."""
if mode not in ('r', 'w', 'x', 'a'):
@@ -1235,6 +1262,12 @@
self.pwd = None
self._comment = b''
self._strict_timestamps = strict_timestamps
+ self.metadata_encoding = metadata_encoding
+
+ # Check that we don't try to write with nonconforming codecs
+ if self.metadata_encoding and mode != 'r':
+ raise ValueError(
+ "metadata_encoding is only supported for reading files")
# Check if we were passed a file-like object
if isinstance(file, os.PathLike):
@@ -1367,13 +1400,13 @@
if self.debug > 2:
print(centdir)
filename = fp.read(centdir[_CD_FILENAME_LENGTH])
- flags = centdir[5]
- if flags & 0x800:
+ flags = centdir[_CD_FLAG_BITS]
+ if flags & _MASK_UTF_FILENAME:
# UTF-8 file names extension
filename = filename.decode('utf-8')
else:
# Historical ZIP filename encoding
- filename = filename.decode('cp437')
+ filename = filename.decode(self.metadata_encoding or 'cp437')
# Create ZipInfo instance to store file information
x = ZipInfo(filename)
x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH])
@@ -1495,8 +1528,6 @@
"""
if mode not in {"r", "w"}:
raise ValueError('open() requires mode "r" or "w"')
- if pwd and not isinstance(pwd, bytes):
- raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__)
if pwd and (mode == "w"):
raise ValueError("pwd is only supported for reading files")
if not self.fp:
@@ -1540,19 +1571,19 @@
if fheader[_FH_EXTRA_FIELD_LENGTH]:
zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
- if zinfo.flag_bits & 0x20:
+ if zinfo.flag_bits & _MASK_COMPRESSED_PATCH:
# Zip 2.7: compressed patched data
raise NotImplementedError("compressed patched data (flag bit 5)")
- if zinfo.flag_bits & 0x40:
+ if zinfo.flag_bits & _MASK_STRONG_ENCRYPTION:
# strong encryption
raise NotImplementedError("strong encryption (flag bit 6)")
- if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800:
+ if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & _MASK_UTF_FILENAME:
# UTF-8 filename
fname_str = fname.decode("utf-8")
else:
- fname_str = fname.decode("cp437")
+ fname_str = fname.decode(self.metadata_encoding or "cp437")
if fname_str != zinfo.orig_filename:
raise BadZipFile(
@@ -1560,10 +1591,12 @@
% (zinfo.orig_filename, fname))
# check for encrypted flag & handle password
- is_encrypted = zinfo.flag_bits & 0x1
+ is_encrypted = zinfo.flag_bits & _MASK_ENCRYPTED
if is_encrypted:
if not pwd:
pwd = self.pwd
+ if pwd and not isinstance(pwd, bytes):
+ raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__)
if not pwd:
raise RuntimeError("File %r is encrypted, password "
"required for extraction" % name)
@@ -1593,16 +1626,17 @@
zinfo.flag_bits = 0x00
if zinfo.compress_type == ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
- zinfo.flag_bits |= 0x02
+ zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1
if not self._seekable:
- zinfo.flag_bits |= 0x08
+ zinfo.flag_bits |= _MASK_USE_DATA_DESCRIPTOR
if not zinfo.external_attr:
zinfo.external_attr = 0o600 << 16 # permissions: ?rw-------
# Compressed size can be larger than uncompressed size
- zip64 = self._allowZip64 and \
- (force_zip64 or zinfo.file_size * 1.05 > ZIP64_LIMIT)
+ zip64 = force_zip64 or (zinfo.file_size * 1.05 > ZIP64_LIMIT)
+ if not self._allowZip64 and zip64:
+ raise LargeZipFile("Filesize would require ZIP64 extensions")
if self._seekable:
self.fp.seek(self.start_dir)
@@ -1744,6 +1778,7 @@
if zinfo.is_dir():
zinfo.compress_size = 0
zinfo.CRC = 0
+ self.mkdir(zinfo)
else:
if compress_type is not None:
zinfo.compress_type = compress_type
@@ -1755,23 +1790,6 @@
else:
zinfo._compresslevel = self.compresslevel
- if zinfo.is_dir():
- with self._lock:
- if self._seekable:
- self.fp.seek(self.start_dir)
- zinfo.header_offset = self.fp.tell() # Start of header bytes
- if zinfo.compress_type == ZIP_LZMA:
- # Compressed data includes an end-of-stream (EOS) marker
- zinfo.flag_bits |= 0x02
-
- self._writecheck(zinfo)
- self._didModify = True
-
- self.filelist.append(zinfo)
- self.NameToInfo[zinfo.filename] = zinfo
- self.fp.write(zinfo.FileHeader(False))
- self.start_dir = self.fp.tell()
- else:
with open(filename, "rb") as src, self.open(zinfo, 'w') as dest:
shutil.copyfileobj(src, dest, 1024*8)
@@ -1816,6 +1834,41 @@
with self.open(zinfo, mode='w') as dest:
dest.write(data)
+ def mkdir(self, zinfo_or_directory_name, mode=511):
+ """Creates a directory inside the zip archive."""
+ if isinstance(zinfo_or_directory_name, ZipInfo):
+ zinfo = zinfo_or_directory_name
+ if not zinfo.is_dir():
+ raise ValueError("The given ZipInfo does not describe a directory")
+ elif isinstance(zinfo_or_directory_name, str):
+ directory_name = zinfo_or_directory_name
+ if not directory_name.endswith("/"):
+ directory_name += "/"
+ zinfo = ZipInfo(directory_name)
+ zinfo.compress_size = 0
+ zinfo.CRC = 0
+ zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16
+ zinfo.file_size = 0
+ zinfo.external_attr |= 0x10
+ else:
+ raise TypeError("Expected type str or ZipInfo")
+
+ with self._lock:
+ if self._seekable:
+ self.fp.seek(self.start_dir)
+ zinfo.header_offset = self.fp.tell() # Start of header bytes
+ if zinfo.compress_type == ZIP_LZMA:
+ # Compressed data includes an end-of-stream (EOS) marker
+ zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1
+
+ self._writecheck(zinfo)
+ self._didModify = True
+
+ self.filelist.append(zinfo)
+ self.NameToInfo[zinfo.filename] = zinfo
+ self.fp.write(zinfo.FileHeader(False))
+ self.start_dir = self.fp.tell()
+
def __del__(self):
"""Call the "close()" method in case the user forgot."""
self.close()
@@ -2372,6 +2425,18 @@
return pathlib.Path(self.at).name or self.filename.name
@property
+ def suffix(self):
+ return pathlib.Path(self.at).suffix or self.filename.suffix
+
+ @property
+ def suffixes(self):
+ return pathlib.Path(self.at).suffixes or self.filename.suffixes
+
+ @property
+ def stem(self):
+ return pathlib.Path(self.at).stem or self.filename.stem
+
+ @property
def filename(self):
return pathlib.Path(self.root.filename).joinpath(self.at)
@@ -2443,11 +2508,15 @@
help='Create zipfile from sources')
group.add_argument('-t', '--test', metavar='<zipfile>',
help='Test if a zipfile is valid')
+ parser.add_argument('--metadata-encoding', metavar='<encoding>',
+ help='Specify encoding of member names for -l, -e and -t')
args = parser.parse_args(args)
+ encoding = args.metadata_encoding
+
if args.test is not None:
src = args.test
- with ZipFile(src, 'r') as zf:
+ with ZipFile(src, 'r', metadata_encoding=encoding) as zf:
badfile = zf.testzip()
if badfile:
print("The following enclosed file is corrupted: {!r}".format(badfile))
@@ -2455,15 +2524,20 @@
elif args.list is not None:
src = args.list
- with ZipFile(src, 'r') as zf:
+ with ZipFile(src, 'r', metadata_encoding=encoding) as zf:
zf.printdir()
elif args.extract is not None:
src, curdir = args.extract
- with ZipFile(src, 'r') as zf:
+ with ZipFile(src, 'r', metadata_encoding=encoding) as zf:
zf.extractall(curdir)
elif args.create is not None:
+ if encoding:
+ print("Non-conforming encodings not supported with -c.",
+ file=sys.stderr)
+ sys.exit(1)
+
zip_name = args.create.pop(0)
files = args.create
diff --git a/common/py3-stdlib/zipimport.py b/common/py3-stdlib/zipimport.py
index 28e4e09..e9da8be 100644
--- a/common/py3-stdlib/zipimport.py
+++ b/common/py3-stdlib/zipimport.py
@@ -63,8 +63,7 @@
# if found, or else read it from the archive.
def __init__(self, path):
if not isinstance(path, str):
- import os
- path = os.fsdecode(path)
+ raise TypeError(f"expected str, not {type(path)!r}")
if not path:
raise ZipImportError('archive path is empty', path=path)
if alt_path_sep:
diff --git a/darwin-x86/bin/aconfig b/darwin-x86/bin/aconfig
index ed2088c..b5ca7cf 100755
--- a/darwin-x86/bin/aconfig
+++ b/darwin-x86/bin/aconfig
Binary files differ
diff --git a/darwin-x86/bin/aidl b/darwin-x86/bin/aidl
index 26110a9..06d447e 100755
--- a/darwin-x86/bin/aidl
+++ b/darwin-x86/bin/aidl
Binary files differ
diff --git a/darwin-x86/bin/bloaty b/darwin-x86/bin/bloaty
index 5030f2d..b731c7b 100755
--- a/darwin-x86/bin/bloaty
+++ b/darwin-x86/bin/bloaty
Binary files differ
diff --git a/darwin-x86/bin/bpfmt b/darwin-x86/bin/bpfmt
index 3fb25ca..10eefc1 100755
--- a/darwin-x86/bin/bpfmt
+++ b/darwin-x86/bin/bpfmt
Binary files differ
diff --git a/darwin-x86/bin/hidl-gen b/darwin-x86/bin/hidl-gen
index 0c7048b..f31c642 100755
--- a/darwin-x86/bin/hidl-gen
+++ b/darwin-x86/bin/hidl-gen
Binary files differ
diff --git a/darwin-x86/bin/hidl-lint b/darwin-x86/bin/hidl-lint
index f4fb471..96ea32c 100755
--- a/darwin-x86/bin/hidl-lint
+++ b/darwin-x86/bin/hidl-lint
Binary files differ
diff --git a/darwin-x86/bin/merge_zips b/darwin-x86/bin/merge_zips
index 8deed88..06ee31d 100755
--- a/darwin-x86/bin/merge_zips
+++ b/darwin-x86/bin/merge_zips
Binary files differ
diff --git a/darwin-x86/bin/one-true-awk b/darwin-x86/bin/one-true-awk
index 47bc8eb..430a601 100755
--- a/darwin-x86/bin/one-true-awk
+++ b/darwin-x86/bin/one-true-awk
Binary files differ
diff --git a/darwin-x86/bin/py3-cmd b/darwin-x86/bin/py3-cmd
index 39f0764..3d31036 100755
--- a/darwin-x86/bin/py3-cmd
+++ b/darwin-x86/bin/py3-cmd
Binary files differ
diff --git a/darwin-x86/bin/py3-launcher-autorun64 b/darwin-x86/bin/py3-launcher-autorun64
index 72f63c6..cf70a32 100755
--- a/darwin-x86/bin/py3-launcher-autorun64
+++ b/darwin-x86/bin/py3-launcher-autorun64
Binary files differ
diff --git a/darwin-x86/bin/py3-launcher64 b/darwin-x86/bin/py3-launcher64
index 2a149a8..ddeb79c 100755
--- a/darwin-x86/bin/py3-launcher64
+++ b/darwin-x86/bin/py3-launcher64
Binary files differ
diff --git a/darwin-x86/bin/soong_zip b/darwin-x86/bin/soong_zip
index 8c14581..437b756 100755
--- a/darwin-x86/bin/soong_zip
+++ b/darwin-x86/bin/soong_zip
Binary files differ
diff --git a/darwin-x86/bin/zip2zip b/darwin-x86/bin/zip2zip
index 2297859..ff6c047 100755
--- a/darwin-x86/bin/zip2zip
+++ b/darwin-x86/bin/zip2zip
Binary files differ
diff --git a/linux-x86/asan/bin/aidl b/linux-x86/asan/bin/aidl
index 4ba4334..60b6a73 100755
--- a/linux-x86/asan/bin/aidl
+++ b/linux-x86/asan/bin/aidl
Binary files differ
diff --git a/linux-x86/bin/aconfig b/linux-x86/bin/aconfig
index d061ae5..13c6bed 100755
--- a/linux-x86/bin/aconfig
+++ b/linux-x86/bin/aconfig
Binary files differ
diff --git a/linux-x86/bin/aidl b/linux-x86/bin/aidl
index 3a9105f..f89e77b 100755
--- a/linux-x86/bin/aidl
+++ b/linux-x86/bin/aidl
Binary files differ
diff --git a/linux-x86/bin/bloaty b/linux-x86/bin/bloaty
index fd95702..a83ee47 100755
--- a/linux-x86/bin/bloaty
+++ b/linux-x86/bin/bloaty
Binary files differ
diff --git a/linux-x86/bin/bpfmt b/linux-x86/bin/bpfmt
index c88dc39..c1dfba5 100755
--- a/linux-x86/bin/bpfmt
+++ b/linux-x86/bin/bpfmt
Binary files differ
diff --git a/linux-x86/bin/merge_zips b/linux-x86/bin/merge_zips
index df12c31..c45200a 100755
--- a/linux-x86/bin/merge_zips
+++ b/linux-x86/bin/merge_zips
Binary files differ
diff --git a/linux-x86/bin/py2-cmd b/linux-x86/bin/py2-cmd
index 31e61f5..4d4336c 100755
--- a/linux-x86/bin/py2-cmd
+++ b/linux-x86/bin/py2-cmd
Binary files differ
diff --git a/linux-x86/bin/py3-cmd b/linux-x86/bin/py3-cmd
index 923f506..712aaf3 100755
--- a/linux-x86/bin/py3-cmd
+++ b/linux-x86/bin/py3-cmd
Binary files differ
diff --git a/linux-x86/bin/py3-launcher-autorun64 b/linux-x86/bin/py3-launcher-autorun64
index df6f2a9..17ebf94 100755
--- a/linux-x86/bin/py3-launcher-autorun64
+++ b/linux-x86/bin/py3-launcher-autorun64
Binary files differ
diff --git a/linux-x86/bin/py3-launcher64 b/linux-x86/bin/py3-launcher64
index 102c8d8..29f758f 100755
--- a/linux-x86/bin/py3-launcher64
+++ b/linux-x86/bin/py3-launcher64
Binary files differ
diff --git a/linux-x86/bin/soong_zip b/linux-x86/bin/soong_zip
index fb0bc86..c1b9cd4 100755
--- a/linux-x86/bin/soong_zip
+++ b/linux-x86/bin/soong_zip
Binary files differ
diff --git a/linux-x86/bin/zip2zip b/linux-x86/bin/zip2zip
index be28aeb..791bfe3 100755
--- a/linux-x86/bin/zip2zip
+++ b/linux-x86/bin/zip2zip
Binary files differ
diff --git a/linux_musl-arm64/bin/aconfig b/linux_musl-arm64/bin/aconfig
index 5d7ca8b..747dd28 100755
--- a/linux_musl-arm64/bin/aconfig
+++ b/linux_musl-arm64/bin/aconfig
Binary files differ
diff --git a/linux_musl-arm64/bin/aidl b/linux_musl-arm64/bin/aidl
index cd762cf..230a2c0 100755
--- a/linux_musl-arm64/bin/aidl
+++ b/linux_musl-arm64/bin/aidl
Binary files differ
diff --git a/linux_musl-arm64/bin/bloaty b/linux_musl-arm64/bin/bloaty
index ed4db12..a585871 100755
--- a/linux_musl-arm64/bin/bloaty
+++ b/linux_musl-arm64/bin/bloaty
Binary files differ
diff --git a/linux_musl-arm64/bin/py2-cmd b/linux_musl-arm64/bin/py2-cmd
index ff51486..d326a13 100755
--- a/linux_musl-arm64/bin/py2-cmd
+++ b/linux_musl-arm64/bin/py2-cmd
Binary files differ
diff --git a/linux_musl-arm64/bin/py3-cmd b/linux_musl-arm64/bin/py3-cmd
index 58fbeac..edacebb 100755
--- a/linux_musl-arm64/bin/py3-cmd
+++ b/linux_musl-arm64/bin/py3-cmd
Binary files differ
diff --git a/linux_musl-arm64/bin/py3-launcher-autorun-static64 b/linux_musl-arm64/bin/py3-launcher-autorun-static64
index fbd6224..61b4f18 100755
--- a/linux_musl-arm64/bin/py3-launcher-autorun-static64
+++ b/linux_musl-arm64/bin/py3-launcher-autorun-static64
Binary files differ
diff --git a/linux_musl-arm64/bin/py3-launcher-autorun64 b/linux_musl-arm64/bin/py3-launcher-autorun64
index fa5b359..bf9423d 100755
--- a/linux_musl-arm64/bin/py3-launcher-autorun64
+++ b/linux_musl-arm64/bin/py3-launcher-autorun64
Binary files differ
diff --git a/linux_musl-arm64/bin/py3-launcher-static64 b/linux_musl-arm64/bin/py3-launcher-static64
index ca5d446..9b07b7c 100755
--- a/linux_musl-arm64/bin/py3-launcher-static64
+++ b/linux_musl-arm64/bin/py3-launcher-static64
Binary files differ
diff --git a/linux_musl-arm64/bin/py3-launcher64 b/linux_musl-arm64/bin/py3-launcher64
index bb4f482..82d29a0 100755
--- a/linux_musl-arm64/bin/py3-launcher64
+++ b/linux_musl-arm64/bin/py3-launcher64
Binary files differ
diff --git a/linux_musl-x86/asan/bin/aidl b/linux_musl-x86/asan/bin/aidl
index 4ba4334..60b6a73 100755
--- a/linux_musl-x86/asan/bin/aidl
+++ b/linux_musl-x86/asan/bin/aidl
Binary files differ
diff --git a/linux_musl-x86/bin/aconfig b/linux_musl-x86/bin/aconfig
index 45a5eac..1f9ceca 100755
--- a/linux_musl-x86/bin/aconfig
+++ b/linux_musl-x86/bin/aconfig
Binary files differ
diff --git a/linux_musl-x86/bin/aidl b/linux_musl-x86/bin/aidl
index 2c9d7ba..9ac09f2 100755
--- a/linux_musl-x86/bin/aidl
+++ b/linux_musl-x86/bin/aidl
Binary files differ
diff --git a/linux_musl-x86/bin/bloaty b/linux_musl-x86/bin/bloaty
index 1fa3f01..51425cb 100755
--- a/linux_musl-x86/bin/bloaty
+++ b/linux_musl-x86/bin/bloaty
Binary files differ
diff --git a/linux_musl-x86/bin/bpfmt b/linux_musl-x86/bin/bpfmt
index c88dc39..c1dfba5 100755
--- a/linux_musl-x86/bin/bpfmt
+++ b/linux_musl-x86/bin/bpfmt
Binary files differ
diff --git a/linux_musl-x86/bin/merge_zips b/linux_musl-x86/bin/merge_zips
index df12c31..c45200a 100755
--- a/linux_musl-x86/bin/merge_zips
+++ b/linux_musl-x86/bin/merge_zips
Binary files differ
diff --git a/linux_musl-x86/bin/py2-cmd b/linux_musl-x86/bin/py2-cmd
index 4789864..725f060 100755
--- a/linux_musl-x86/bin/py2-cmd
+++ b/linux_musl-x86/bin/py2-cmd
Binary files differ
diff --git a/linux_musl-x86/bin/py3-cmd b/linux_musl-x86/bin/py3-cmd
index 5708f81..502848c 100755
--- a/linux_musl-x86/bin/py3-cmd
+++ b/linux_musl-x86/bin/py3-cmd
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher-autorun-static64 b/linux_musl-x86/bin/py3-launcher-autorun-static64
index aa28921..aa80976 100755
--- a/linux_musl-x86/bin/py3-launcher-autorun-static64
+++ b/linux_musl-x86/bin/py3-launcher-autorun-static64
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher-autorun64 b/linux_musl-x86/bin/py3-launcher-autorun64
index d95e562..0cbbb4a 100755
--- a/linux_musl-x86/bin/py3-launcher-autorun64
+++ b/linux_musl-x86/bin/py3-launcher-autorun64
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher-static64 b/linux_musl-x86/bin/py3-launcher-static64
index a258742..a62418b 100755
--- a/linux_musl-x86/bin/py3-launcher-static64
+++ b/linux_musl-x86/bin/py3-launcher-static64
Binary files differ
diff --git a/linux_musl-x86/bin/py3-launcher64 b/linux_musl-x86/bin/py3-launcher64
index 46b2d7e..1430bc2 100755
--- a/linux_musl-x86/bin/py3-launcher64
+++ b/linux_musl-x86/bin/py3-launcher64
Binary files differ
diff --git a/linux_musl-x86/bin/soong_zip b/linux_musl-x86/bin/soong_zip
index fb0bc86..c1b9cd4 100755
--- a/linux_musl-x86/bin/soong_zip
+++ b/linux_musl-x86/bin/soong_zip
Binary files differ
diff --git a/linux_musl-x86/bin/zip2zip b/linux_musl-x86/bin/zip2zip
index be28aeb..791bfe3 100755
--- a/linux_musl-x86/bin/zip2zip
+++ b/linux_musl-x86/bin/zip2zip
Binary files differ
diff --git a/manifest.xml b/manifest.xml
index 7a1330a..3303cc3 100644
--- a/manifest.xml
+++ b/manifest.xml
@@ -5,11 +5,11 @@
<default revision="build-tools-release" remote="aosp" sync-j="4" />
- <project path="build/make" name="platform/build" revision="a9b3156dcbf5590f7573fbd5623b39071b38a9c8" upstream="build-tools-release">
+ <project path="build/make" name="platform/build" revision="c98607cd907d1aaec40ecb5c16d2331a73cdc8c1" upstream="build-tools-release">
<linkfile dest="build/tools" src="tools" />
</project>
- <project path="build/bazel" name="platform/build/bazel" groups="pdk" revision="2dc081ff716e39104da88a8994930cc761f1f007" upstream="build-tools-release">
+ <project path="build/bazel" name="platform/build/bazel" groups="pdk" revision="421921943d6c1df3a4f79b40794f5249a9ffa59a" upstream="build-tools-release">
<linkfile dest="WORKSPACE" src="bazel.WORKSPACE" />
<linkfile dest="BUILD" src="bazel.BUILD" />
@@ -17,9 +17,9 @@
<project path="build/bazel_common_rules" name="platform/build/bazel_common_rules" groups="pdk" revision="f0319530fc418626aa88e842b0f8abd657e8342f" upstream="build-tools-release" />
- <project path="build/blueprint" name="platform/build/blueprint" revision="19b733350e1d831280ac68ee019f33a50dd6e2c7" upstream="build-tools-release" />
+ <project path="build/blueprint" name="platform/build/blueprint" revision="f136df1605554043027656e4d4fe2cc05ca4c9a6" upstream="build-tools-release" />
- <project path="build/soong" name="platform/build/soong" revision="f989aea93eb648b7055c4515064b3fe9bcc1c982" upstream="build-tools-release">
+ <project path="build/soong" name="platform/build/soong" revision="d0dd33a83f7efaf931e0db9dbbf411f89e129eae" upstream="build-tools-release">
<linkfile dest="Android.bp" src="root.bp" />
<linkfile dest="bootstrap.bash" src="bootstrap.bash" />
@@ -29,25 +29,25 @@
<project path="external/golang-protobuf" name="platform/external/golang-protobuf" revision="24b05b92df7699d6f74c55bb5230cdb0812c1293" upstream="build-tools-release" />
- <project path="packages/modules/common" name="platform/packages/modules/common" groups="pdk-cw-fs,pdk-fs" revision="f8defbca3e5f4f2a136ccd8b7442616726ddc296" upstream="build-tools-release" />
+ <project path="packages/modules/common" name="platform/packages/modules/common" groups="pdk-cw-fs,pdk-fs" revision="525416800b898aeab918c51a746a125b9cf2b9a8" upstream="build-tools-release" />
- <project path="prebuilts/bazel/common" name="platform/prebuilts/bazel/common" groups="pdk" clone-depth="1" revision="3b7cc6510ad23cdb8e5a892c164fc68609b42882" upstream="build-tools-release" />
+ <project path="prebuilts/bazel/common" name="platform/prebuilts/bazel/common" groups="pdk" clone-depth="1" revision="463afe79616bbfcbe62d0449f03e2b82c7227e7a" upstream="build-tools-release" />
- <project path="prebuilts/bazel/darwin-x86_64" name="platform/prebuilts/bazel/darwin-x86_64" groups="notdefault,platform-darwin,pdk" clone-depth="1" revision="6cbc46ba1157640cd71a61881cad3e961f33e1c6" upstream="build-tools-release" />
+ <project path="prebuilts/bazel/darwin-x86_64" name="platform/prebuilts/bazel/darwin-x86_64" groups="notdefault,platform-darwin,pdk" clone-depth="1" revision="48c716fe4893e089ca4eaf3cf606e53af3f4c957" upstream="build-tools-release" />
- <project path="prebuilts/bazel/linux-x86_64" name="platform/prebuilts/bazel/linux-x86_64" groups="linux,pdk" clone-depth="1" revision="8455891410f091b431c0781158431fcc40beeb17" upstream="build-tools-release" />
+ <project path="prebuilts/bazel/linux-x86_64" name="platform/prebuilts/bazel/linux-x86_64" groups="linux,pdk" clone-depth="1" revision="cb8ee3865d52011cf508b2affc7e57338c772532" upstream="build-tools-release" />
- <project path="prebuilts/build-tools" name="platform/prebuilts/build-tools" clone-depth="1" revision="0b38a34db5197a96284bc0e629c138bf4dab1f9e" upstream="build-tools-release" />
+ <project path="prebuilts/build-tools" name="platform/prebuilts/build-tools" clone-depth="1" revision="b7130172036693489ca5f95b5a2b8cff17082468" upstream="build-tools-release" />
- <project path="prebuilts/remoteexecution-client" name="platform/prebuilts/remoteexecution-client" groups="pdk" clone-depth="1" revision="ec6ae824845ad36bfb6566a041579487895dc9b2" upstream="build-tools-release" />
+ <project path="prebuilts/remoteexecution-client" name="platform/prebuilts/remoteexecution-client" groups="pdk" clone-depth="1" revision="4d019aa3c1e54c5991c2aed2660749f210b0986b" upstream="build-tools-release" />
- <project path="prebuilts/clang/host/linux-x86" name="platform/prebuilts/clang/host/linux-x86" groups="linux" clone-depth="1" revision="c3b82555a7bec944b7a0cf2ddc1ea8d31d668bea" upstream="build-tools-release" />
+ <project path="prebuilts/clang/host/linux-x86" name="platform/prebuilts/clang/host/linux-x86" groups="linux" clone-depth="1" revision="e610e08d073315afaa24991d389f21b1012309e7" upstream="build-tools-release" />
- <project path="prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8" name="platform/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8" clone-depth="1" revision="bc5eaca4e7f4e38d0b97ca72e982a46092c268bf" upstream="build-tools-release" />
+ <project path="prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8" name="platform/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8" clone-depth="1" revision="ed3c840a1c6682d6d9eaa813e2d83512f1074ac1" upstream="build-tools-release" />
<project path="prebuilts/go/linux-x86" name="platform/prebuilts/go/linux-x86" groups="linux" clone-depth="1" revision="12fd634fe8d9a2a4917fb01081fef751b1a9e5f6" upstream="build-tools-release" />
- <project path="prebuilts/clang/host/darwin-x86" name="platform/prebuilts/clang/host/darwin-x86" groups="notdefault,platform-darwin" clone-depth="1" revision="4fa78ab41509e6aaedfebcce586c7ba03cbc3ee2" upstream="build-tools-release" />
+ <project path="prebuilts/clang/host/darwin-x86" name="platform/prebuilts/clang/host/darwin-x86" groups="notdefault,platform-darwin" clone-depth="1" revision="80d9524f24940a2a787cf9b298f09b4e1543ca72" upstream="build-tools-release" />
<project path="prebuilts/go/darwin-x86" name="platform/prebuilts/go/darwin-x86" groups="notdefault,platform-darwin" clone-depth="1" revision="0f4809f7207c59639b51fdf1e999d3c3493b4c09" upstream="build-tools-release" />
@@ -63,23 +63,23 @@
<project path="prebuilts/clang-tools" name="platform/prebuilts/clang-tools" clone-depth="1" revision="e1e81d170f78c9828bca653eed0039660cda8979" upstream="build-tools-release" />
- <project path="prebuilts/misc" name="platform/prebuilts/misc" clone-depth="1" revision="c7f9e386f5331670e9fbebb0892b19eb399c8b35" upstream="build-tools-release" />
+ <project path="prebuilts/misc" name="platform/prebuilts/misc" clone-depth="1" revision="bd8f7e845a0fe2a2687b3744b593535408381146" upstream="build-tools-release" />
<project path="prebuilts/tools" name="platform/prebuilts/tools" groups="pdk,tools" clone-depth="1" revision="957bd31a0a5950db464573d35b9df3f1cb02e609" upstream="build-tools-release" />
- <project path="bionic" name="platform/bionic" revision="be836ada99c7231c452d9ca4802a9f7ad6f1c58e" upstream="build-tools-release" />
+ <project path="bionic" name="platform/bionic" revision="1a9168c59d1be91b989d381fad73d98661c02bad" upstream="build-tools-release" />
- <project path="development" name="platform/development" revision="5a5878568928fbd4beab4b350b8a435572bce080" upstream="build-tools-release" />
+ <project path="development" name="platform/development" revision="56d6d10b8fceb3aaaca3e2d4833c3d9949ccf0c3" upstream="build-tools-release" />
<project path="external/arm-optimized-routines" name="platform/external/arm-optimized-routines" revision="6e7953c622623e9799aa7a778633224aaacbd24a" upstream="build-tools-release" />
<project path="external/auto" name="platform/external/auto" revision="af8aa9ab6e0f850b3320d06a80e7dc5027f07470" upstream="build-tools-release" />
- <project path="external/abseil-cpp" name="platform/external/abseil-cpp" revision="f1a6e37ec5da8a99ce8085b3b727cdded6c26525" upstream="build-tools-release" />
+ <project path="external/abseil-cpp" name="platform/external/abseil-cpp" revision="136a02e2c31e476a78d9707c682dc9cbaa69fe0f" upstream="build-tools-release" />
<project path="external/bazelbuild-rules_license" name="platform/external/bazelbuild-rules_license" revision="80ac2b0a3409933d28fc8392ae0f057b3108571e" upstream="build-tools-release" />
- <project path="external/boringssl" name="platform/external/boringssl" revision="534dbc23de00cca4dc23a1b81f49d0e8334370e1" upstream="build-tools-release" />
+ <project path="external/boringssl" name="platform/external/boringssl" revision="f72078fbc2ccad53d59feacca8fb281c09e05871" upstream="build-tools-release" />
<project path="external/brotli" name="platform/external/brotli" revision="9809046e22e10d26515f2b60b43671d776d077c8" upstream="build-tools-release" />
@@ -135,7 +135,7 @@
<project path="external/kythe" name="platform/external/kythe" revision="cb0855583983cfa552a04afc0bd650d1b7a85d25" upstream="build-tools-release" />
- <project path="external/libcap" name="platform/external/libcap" revision="f283766a08724d6e62b35ade2efb79d2784fff30" upstream="build-tools-release" />
+ <project path="external/libcap" name="platform/external/libcap" revision="c5d4aa6bcb8ca30cc49bf434b88896b42efb9c2f" upstream="build-tools-release" />
<project path="external/libcxx" name="platform/external/libcxx" revision="9bff73984e755012fbe6703775b84d3be102710d" upstream="build-tools-release" />
@@ -161,7 +161,7 @@
<project path="external/pcre" name="platform/external/pcre" revision="14ca9f0ba43fcc8abd2bbbaf75dd19176f027614" upstream="build-tools-release" />
- <project path="external/protobuf" name="platform/external/protobuf" revision="500fc860855bcf6caf55ce6bd2bf594a5bee8309" upstream="build-tools-release" />
+ <project path="external/protobuf" name="platform/external/protobuf" revision="7dfdd112a5759e387c915617e9b8a88019262524" upstream="build-tools-release" />
<project path="external/safe-iop" name="platform/external/safe-iop" revision="b805514f31a231a0e78a18f296c0454fcadead1a" upstream="build-tools-release" />
@@ -173,13 +173,13 @@
<project path="external/starlark-go" name="platform/external/starlark-go" revision="1193e1a3860f51a54300652edaf1668ade9f986d" upstream="build-tools-release" />
- <project path="external/tinyxml2" name="platform/external/tinyxml2" revision="54e4a18bdf47da15787bb1490d4494a75dbd3fb4" upstream="build-tools-release" />
+ <project path="external/tinyxml2" name="platform/external/tinyxml2" revision="024d8cda3270089c93adb19d7be8c90fe3b485a4" upstream="build-tools-release" />
<project path="external/zlib" name="platform/external/zlib" revision="a0bfa0617ae7f77ef873b1addf049580ca1339af" upstream="build-tools-release" />
<project path="external/zopfli" name="platform/external/zopfli" revision="553c25fd956818afbae5e5f90ce2ddaa8863293a" upstream="build-tools-release" />
- <project path="system/core" name="platform/system/core" revision="6a9ddd32c90c613b7fe9191002b3474a83629c65" upstream="build-tools-release" />
+ <project path="system/core" name="platform/system/core" revision="49c6ff46fdcb46cb20e963cf39eed83f02c875b5" upstream="build-tools-release" />
<project path="system/libbase" name="platform/system/libbase" revision="c6b373a88ea57e3da7028f1885ec983f0de62b50" upstream="build-tools-release" />
@@ -187,17 +187,17 @@
<project path="system/libziparchive" name="platform/system/libziparchive" revision="20a3cba9983e53aed8cd43e858dce2667916279b" upstream="build-tools-release" />
- <project path="system/logging" name="platform/system/logging" revision="77811469a1d438f463bbafb4fd402f37878d1e37" upstream="build-tools-release" />
+ <project path="system/logging" name="platform/system/logging" revision="09e18b9361ca60554b022509dde0c73e9dc01604" upstream="build-tools-release" />
<project path="system/unwinding" name="platform/system/unwinding" revision="499a4930b0e472a2efbe07c44a4d24222e96902b" upstream="build-tools-release" />
<project path="system/tools/xsdc" name="platform/system/tools/xsdc" revision="23a00ac8b0c2a7310039e85dc53ef070d41bd95e" upstream="build-tools-release" />
- <project path="test/app_compat/csuite" name="platform/test/app_compat/csuite" revision="909c219a93a2d0aa75aec0a2cbcafd4378a9ef67" upstream="build-tools-release" />
+ <project path="test/app_compat/csuite" name="platform/test/app_compat/csuite" revision="8f51783111c6ea2b14c33cd07b3702a11536a258" upstream="build-tools-release" />
- <project path="system/apex" name="platform/system/apex" groups="pdk" revision="5401f086ac4ede616adcbd0bf8d344281f4417fa" upstream="build-tools-release" />
+ <project path="system/apex" name="platform/system/apex" groups="pdk" revision="6f1871c5a7d71a0915663741eda53002a013b3fa" upstream="build-tools-release" />
- <project path="art" name="platform/art" revision="e9e8360d75cc9bb74fec425d6f53894995c6d48f" upstream="build-tools-release" />
+ <project path="art" name="platform/art" revision="d13dcf262667e83041c935bbd9b3a7b5ed48a999" upstream="build-tools-release" />
<project path="build/kati" name="platform/build/kati" revision="c71a989f5a512ee947facbc8dbfe3c4b122a99b6" upstream="build-tools-release" />
@@ -231,13 +231,13 @@
<project path="external/python/cpython2" name="platform/external/python/cpython2" revision="0cebe3b6c590c3b55798df694a4b7f67dbb88e9f" upstream="build-tools-release" />
- <project path="external/python/cpython3" name="platform/external/python/cpython3" revision="864696e4ea578f5d65145878ed87f61907fccbb4" upstream="build-tools-release" />
+ <project path="external/python/cpython3" name="platform/external/python/cpython3" revision="ffb26860a20c043d73daed190d2c1095bdbc18b8" upstream="build-tools-release" />
<project path="external/toybox" name="platform/external/toybox" revision="2335ac043d730a7e486453e96c60014d868a05bf" upstream="build-tools-release" />
<project path="external/turbine" name="platform/external/turbine" revision="794370a668c80b0b27580f5e402285da3c9273c1" upstream="build-tools-release" />
- <project path="system/tools/aidl" name="platform/system/tools/aidl" revision="9846b584cf1c9385b112fce00cd06b958492ee02" upstream="build-tools-release" />
+ <project path="system/tools/aidl" name="platform/system/tools/aidl" revision="3545d6b23e3c61d4f228e77e2af7764a7da0c355" upstream="build-tools-release" />
<project path="system/tools/hidl" name="platform/system/tools/hidl" revision="9da85f4d392d5839ecbb33e3512597dd3d49f434" upstream="build-tools-release" />
@@ -281,7 +281,7 @@
<project path="external/rust/crates/async-task" name="platform/external/rust/crates/async-task" groups="pdk" revision="d0024473d3e1c551b753946af606563fc2f6a35b" upstream="build-tools-release" />
- <project path="external/rust/crates/async-trait" name="platform/external/rust/crates/async-trait" groups="pdk" revision="5da30f30bbb506eb0a83e8672101db068c698769" upstream="build-tools-release" />
+ <project path="external/rust/crates/async-trait" name="platform/external/rust/crates/async-trait" groups="pdk" revision="1e82878ad4a19f26c9f2b87ca437e7068e91d26b" upstream="build-tools-release" />
<project path="external/rust/crates/atomic" name="platform/external/rust/crates/atomic" groups="pdk" revision="c1ba845c9c0c327cd6927407259f54198701ad84" upstream="build-tools-release" />
@@ -819,5 +819,5 @@
<project path="external/rust/pica" name="platform/external/rust/pica" groups="pdk" revision="fe1453f73fd7dbfe7f3696dff0fa0a376250b648" upstream="build-tools-release" />
- <project path="prebuilts/rust" name="platform/prebuilts/rust" groups="pdk" clone-depth="1" revision="9dc519d8572c47295a3e4c5dfb717929e04babfe" upstream="build-tools-release" />
+ <project path="prebuilts/rust" name="platform/prebuilts/rust" groups="pdk" clone-depth="1" revision="be3c8e4d6c9d76c6e7da86d12c792aa20fee1a86" upstream="build-tools-release" />
</manifest>
