Upgrade python/httplib2 to v0.18.1 am: f6b4dcdd5a am: 312ef219b4 am: 8cef85bf28 am: 43ecfa8e0c
Change-Id: I624e6736281ecf14c29eb4603f1cbea7579c4180
diff --git a/.travis.yml b/.travis.yml
index 395c507..ed1aa72 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,7 @@
env:
global:
- - pip_install_common='pip>=9.0 setuptools>=36.2 wheel>=0.30'
+ - pip_install_common='pip>=9.0 setuptools>=43.0 wheel>=0.30'
python:
- 2.7
- 3.5
diff --git a/CHANGELOG b/CHANGELOG
index ba20b24..2db1cc0 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,26 @@
+0.18.1
+
+ explicit build-backend workaround for pip build isolation bug
+ "AttributeError: 'module' object has no attribute '__legacy__'" on pip install
+ https://github.com/httplib2/httplib2/issues/169
+
+0.18.0
+
+ IMPORTANT security vulnerability CWE-93 CRLF injection
+ Force %xx quote of space, CR, LF characters in uri.
+ Special thanks to Recar https://github.com/Ciyfly for discrete notification.
+ https://cwe.mitre.org/data/definitions/93.html
+
+0.17.4
+
+ Ship test suite in source dist
+ https://github.com/httplib2/httplib2/pull/168
+
+0.17.3
+
+ IronPython2.7: relative import iri2uri fixes ImportError
+ https://github.com/httplib2/httplib2/pull/163
+
0.17.2
python3 + debug + IPv6 disabled: https raised
diff --git a/MANIFEST.in b/MANIFEST.in
index 12c4cc7..412def6 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,11 @@
recursive-include python2 *.py *.txt
recursive-include python3 *.py *.txt
+graft test
+graft tests
+include *.md
+include CHANGELOG
+include LICENSE
include python2/httplib2/test/*.txt
include requirements*.txt
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/METADATA b/METADATA
index f393882..ac99305 100644
--- a/METADATA
+++ b/METADATA
@@ -9,11 +9,11 @@
type: GIT
value: "https://github.com/httplib2/httplib2.git"
}
- version: "v0.17.2"
+ version: "v0.18.1"
license_type: NOTICE
last_upgrade_date {
year: 2020
- month: 4
- day: 13
+ month: 5
+ day: 20
}
}
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..5eb3903
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,19 @@
+# Security Policy
+
+## Supported Versions
+
+master branch and latest release get priority support. You should expect all known problems fixed in master.
+
+All other released versions receive security updates per request.
+If you use some old version and can not upgrade for any or no reason, ask for security update release, most likely you will get it.
+
+## Reporting a Vulnerability
+
+Contact current maintainers. At 2020-05: temotor@gmail.com or https://t.me/temotor
+If that doesn't work, open Github issue just asking for private communication channel.
+
+This is volunteer maintained project, all issues are processed on best effort basis, no deadlines promised. Of course, security vulnerabilities get priority over regular issues.
+
+You can expect fame in history or maybe you prefer anonymity - say what you prefer.
+
+Thank you for responsible handling of security problems. Your attention and effort are appreciated.
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..5f7cbbd
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,9 @@
+[build-system]
+requires = ["setuptools >= 40.8.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 121
+
+[tool.check-manifest]
+ignore = [".travis.yml", "script/*", "*.tex"]
diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py
index 9057d1f..f35ba48 100644
--- a/python2/httplib2/__init__.py
+++ b/python2/httplib2/__init__.py
@@ -19,7 +19,7 @@
"Alex Yu",
]
__license__ = "MIT"
-__version__ = '0.17.2'
+__version__ = "0.18.1"
import base64
import calendar
@@ -129,7 +129,7 @@
_ssl_wrap_socket = _ssl_wrap_socket_unsupported
if sys.version_info >= (2, 3):
- from iri2uri import iri2uri
+ from .iri2uri import iri2uri
else:
def iri2uri(uri):
@@ -1985,6 +1985,9 @@
headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__
uri = iri2uri(uri)
+ # Prevent CWE-75 space injection to manipulate request via part of uri.
+ # Prevent CWE-93 CRLF injection to modify headers via part of uri.
+ uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A")
(scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py
index 135c6f6..cf2db60 100644
--- a/python3/httplib2/__init__.py
+++ b/python3/httplib2/__init__.py
@@ -15,7 +15,7 @@
"Alex Yu",
]
__license__ = "MIT"
-__version__ = '0.17.2'
+__version__ = "0.18.1"
import base64
import calendar
@@ -1790,6 +1790,9 @@
headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__
uri = iri2uri(uri)
+ # Prevent CWE-75 space injection to manipulate request via part of uri.
+ # Prevent CWE-93 CRLF injection to modify headers via part of uri.
+ uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A")
(scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
diff --git a/script/release b/script/release
index 0f98e3e..a2ff80d 100755
--- a/script/release
+++ b/script/release
@@ -45,9 +45,9 @@
last_tag=$(git tag --sort=-version:refname |head -n1)
last_tag=${last_tag##v}
version_replace="${last_tag}.post$(date -u +%y%m%d%H%M)"
- update_version "setup.py" "s/VERSION =.+/VERSION = '$version_replace'/"
- update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/"
- update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/"
+ update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_replace\"/"
+ update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/"
+ update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/"
version_check "$version_replace"
fi
}
@@ -90,10 +90,11 @@
local venv=./venv-release
if [[ ! -d "$venv" ]] ; then
virtualenv $venv
- $venv/bin/pip install -U pip setuptools wheel twine
+ $venv/bin/pip install -U check-manifest pip 'setuptools>=43.0' wheel twine
fi
$venv/bin/python setup.py clean --all
$venv/bin/python setup.py sdist bdist_wheel
+ $venv/bin/check-manifest || echo "FIXME check-manifest" >&2
if confirm "Upload to PyPI? Use in special situation, normally CI (Travis) will upload to PyPI. [yN] " ; then
$venv/bin/twine upload dist/* || exit 1
@@ -132,9 +133,9 @@
fi
echo "Next version: '$version_next'" >&2
- update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/"
- update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/"
- update_version "setup.py" "s/VERSION =.+/VERSION = '$version_next'/"
+ update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/"
+ update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/"
+ update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_next\"/"
confirm "Confirm changes? [yN] " || exit 1
}
@@ -142,8 +143,8 @@
update_version() {
local path="$1"
local sed_expr="$2"
- # sed -E --in-place='' -e "s/VERSION =.+/VERSION = '$version_replace'/" setup.py
- # sed -E --in-place='' -e "s/__version__ =.+/__version__ = '$version_replace'/" python2/httplib2/__init__.py python3/httplib2/__init__.py
+ # sed -E --in-place='' -e "s/VERSION =.+/VERSION = \"$version_replace\"/" setup.py
+ # sed -E --in-place='' -e "s/__version__ =.+/__version__ = \"$version_replace\"/" python2/httplib2/__init__.py python3/httplib2/__init__.py
echo "Updating file '$path'" >&2
if ! sed -E --in-place='' -e "$sed_expr" "$path" ; then
echo "sed error $?" >&2
@@ -209,7 +210,7 @@
version_check() {
local need=$1
- local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '" |cut -d\= -f2)
+ local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '\"" |cut -d\= -f2)
local version_py2=$(cd python2 ; python2 -Es -c 'import httplib2;print(httplib2.__version__)')
local version_py3=$(cd python3 ; python3 -Es -c 'import httplib2;print(httplib2.__version__)')
if [[ "$version_setup" != "$need" ]] ; then
diff --git a/setup.py b/setup.py
index 9140845..b66d24e 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
import sys
pkgdir = {"": "python%s" % sys.version_info[0]}
-VERSION = '0.17.2'
+VERSION = "0.18.1"
# `python setup.py test` uses existing Python environment, no virtualenv, no pip.
diff --git a/tests/__init__.py b/tests/__init__.py
index a15db9e..02a3ecf 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -75,7 +75,7 @@
chunk = b""
else:
chunk = self._sock.recv(8 << 10)
- # print('!!! recv', chunk)
+ # print("!!! recv", chunk)
if not chunk:
self._end = True
if untilend:
diff --git a/tests/test_http.py b/tests/test_http.py
index df99016..f61992c 100644
--- a/tests/test_http.py
+++ b/tests/test_http.py
@@ -703,3 +703,33 @@
response, content = http.request(uri, "GET")
assert response.status == 301
assert response.previous is None
+
+
+def test_cwe93_inject_crlf():
+ # https://cwe.mitre.org/data/definitions/93.html
+ # GET /?q= HTTP/1.1 <- injected "HTTP/1.1" from attacker
+ # injected: attack
+ # ignore-http: HTTP/1.1 <- nominal "HTTP/1.1" from library
+ # Host: localhost:57285
+ http = httplib2.Http()
+ with tests.server_reflect() as uri:
+ danger_url = urllib.parse.urljoin(
+ uri, "?q= HTTP/1.1\r\ninjected: attack\r\nignore-http:"
+ )
+ response, content = http.request(danger_url, "GET")
+ assert response.status == 200
+ req = tests.HttpRequest.from_bytes(content)
+ assert req.headers.get("injected") is None
+
+
+def test_inject_space():
+ # Injecting space into request line is precursor to CWE-93 and possibly other injections
+ http = httplib2.Http()
+ with tests.server_reflect() as uri:
+ # "\r\nignore-http:" suffix is nuance for current server implementation
+ # please only pay attention to space after "?q="
+ danger_url = urllib.parse.urljoin(uri, "?q= HTTP/1.1\r\nignore-http:")
+ response, content = http.request(danger_url, "GET")
+ assert response.status == 200
+ req = tests.HttpRequest.from_bytes(content)
+ assert req.uri == "/?q=%20HTTP/1.1%0D%0Aignore-http:"
