bpo-43014: Improve performance of tokenize.tokenize by 20-30%
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1aee21b..42c1f10 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -27,6 +27,7 @@
from builtins import open as _builtin_open
from codecs import lookup, BOM_UTF8
import collections
+import functools
from io import TextIOWrapper
import itertools as _itertools
import re
@@ -95,6 +96,7 @@ def _all_string_prefixes():
result.add(''.join(u))
return result
+@functools.lru_cache
def _compile(expr):
return re.compile(expr, re.UNICODE)
diff --git a/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst
new file mode 100644
index 0000000..02898e4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst
@@ -0,0 +1 @@
+Improve performance of :mod:`tokenize` by 20-30%. Patch by Anthony Sottile.
