Upgrade python/httplib2 to v0.17.0
Test: None
Change-Id: I4189cf870ba80629316fbe58c56b996fa0a082d5
diff --git a/CHANGELOG b/CHANGELOG
index 07fb949..1dee4c4 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,18 @@
+0.17.0
+
+ feature: Http().redirect_codes set, works after follow(_all)_redirects check
+ This allows one line workaround for old gcloud library that uses 308
+ response without redirect semantics.
+ https://github.com/httplib2/httplib2/issues/156
+
+0.16.0
+
+ IMPORTANT cache invalidation change, fix 307 keep method, add 308 Redirects
+ https://github.com/httplib2/httplib2/issues/151
+
+ proxy: username/password as str compatible with pysocks
+ https://github.com/httplib2/httplib2/issues/154
+
0.15.0
python2: regression in connect() error handling
diff --git a/METADATA b/METADATA
index 8b3c1c6..605407f 100644
--- a/METADATA
+++ b/METADATA
@@ -9,10 +9,10 @@
type: GIT
value: "https://github.com/httplib2/httplib2.git"
}
- version: "v0.15.0"
+ version: "v0.17.0"
last_upgrade_date {
- year: 2019
- month: 12
- day: 23
+ year: 2020
+ month: 2
+ day: 1
}
}
diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py
index c8302eb..f32accf 100644
--- a/python2/httplib2/__init__.py
+++ b/python2/httplib2/__init__.py
@@ -19,7 +19,7 @@
"Alex Yu",
]
__license__ = "MIT"
-__version__ = '0.15.0'
+__version__ = '0.17.0'
import base64
import calendar
@@ -291,6 +291,12 @@
"upgrade",
]
+# https://tools.ietf.org/html/rfc7231#section-8.1.3
+SAFE_METHODS = ("GET", "HEAD") # TODO add "OPTIONS", "TRACE"
+
+# To change, assign to `Http().redirect_codes`
+REDIRECT_CODES = frozenset((300, 301, 302, 303, 307, 308))
+
def _get_end2end_headers(response):
hopbyhop = list(HOP_BY_HOP)
@@ -1175,9 +1181,9 @@
host = self.host
port = self.port
-
+
socket_err = None
-
+
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
try:
@@ -1353,9 +1359,9 @@
host = self.host
port = self.port
-
+
socket_err = None
-
+
address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
for family, socktype, proto, canonname, sockaddr in address_info:
try:
@@ -1661,10 +1667,14 @@
# If set to False then no redirects are followed, even safe ones.
self.follow_redirects = True
+ self.redirect_codes = REDIRECT_CODES
+
# Which HTTP methods do we apply optimistic concurrency to, i.e.
# which methods get an "if-match:" etag header added to them.
self.optimistic_concurrency_methods = ["PUT", "PATCH"]
+ self.safe_methods = list(SAFE_METHODS)
+
# If 'follow_redirects' is True, and this is set to True then
# all redirecs are followed, including unsafe ones.
self.follow_all_redirects = False
@@ -1858,10 +1868,10 @@
if (
self.follow_all_redirects
- or (method in ["GET", "HEAD"])
- or response.status == 303
+ or method in self.safe_methods
+ or response.status in (303, 308)
):
- if self.follow_redirects and response.status in [300, 301, 302, 303, 307]:
+ if self.follow_redirects and response.status in self.redirect_codes:
# Pick out the location header and basically start from the beginning
# remembering first to strip the ETag header and decrement our 'depth'
if redirections:
@@ -1881,7 +1891,7 @@
response["location"] = urlparse.urljoin(
absolute_uri, location
)
- if response.status == 301 and method in ["GET", "HEAD"]:
+ if response.status == 308 or (response.status == 301 and method in self.safe_methods):
response["-x-permanent-redirect-url"] = response["location"]
if "content-location" not in response:
response["content-location"] = absolute_uri
@@ -1918,7 +1928,7 @@
response,
content,
)
- elif response.status in [200, 203] and method in ["GET", "HEAD"]:
+ elif response.status in [200, 203] and method in self.safe_methods:
# Don't cache 206's since we aren't going to handle byte range requests
if "content-location" not in response:
response["content-location"] = absolute_uri
@@ -2018,6 +2028,7 @@
headers["accept-encoding"] = "gzip, deflate"
info = email.Message.Message()
+ cachekey = None
cached_value = None
if self.cache:
cachekey = defrag_uri.encode("utf-8")
@@ -2038,8 +2049,6 @@
self.cache.delete(cachekey)
cachekey = None
cached_value = None
- else:
- cachekey = None
if (
method in self.optimistic_concurrency_methods
@@ -2051,13 +2060,15 @@
# http://www.w3.org/1999/04/Editing/
headers["if-match"] = info["etag"]
- if method not in ["GET", "HEAD"] and self.cache and cachekey:
- # RFC 2616 Section 13.10
+ # https://tools.ietf.org/html/rfc7234
+ # A cache MUST invalidate the effective Request URI as well as [...] Location and Content-Location
+ # when a non-error status code is received in response to an unsafe request method.
+ if self.cache and cachekey and method not in self.safe_methods:
self.cache.delete(cachekey)
# Check the vary header in the cache to see if this request
# matches what varies in the cache.
- if method in ["GET", "HEAD"] and "vary" in info:
+ if method in self.safe_methods and "vary" in info:
vary = info["vary"]
vary_headers = vary.lower().replace(" ", "").split(",")
for header in vary_headers:
@@ -2068,11 +2079,14 @@
break
if (
- cached_value
- and method in ["GET", "HEAD"]
- and self.cache
+ self.cache
+ and cached_value
+ and (method in self.safe_methods or info["status"] == "308")
and "range" not in headers
):
+ redirect_method = method
+ if info["status"] not in ("307", "308"):
+ redirect_method = "GET"
if "-x-permanent-redirect-url" in info:
# Should cached permanent redirects be counted in our redirection count? For now, yes.
if redirections <= 0:
@@ -2083,7 +2097,7 @@
)
(response, new_content) = self.request(
info["-x-permanent-redirect-url"],
- method="GET",
+ method=redirect_method,
headers=headers,
redirections=redirections - 1,
)
diff --git a/python2/httplib2/socks.py b/python2/httplib2/socks.py
index 5cef776..71eb4eb 100644
--- a/python2/httplib2/socks.py
+++ b/python2/httplib2/socks.py
@@ -238,7 +238,15 @@
headers - Additional or modified headers for the proxy connect
request.
"""
- self.__proxy = (proxytype, addr, port, rdns, username, password, headers)
+ self.__proxy = (
+ proxytype,
+ addr,
+ port,
+ rdns,
+ username.encode() if username else None,
+ password.encode() if password else None,
+ headers,
+ )
def __negotiatesocks5(self, destaddr, destport):
"""__negotiatesocks5(self,destaddr,destport)
diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py
index d8c3d34..6467c79 100644
--- a/python3/httplib2/__init__.py
+++ b/python3/httplib2/__init__.py
@@ -15,7 +15,7 @@
"Alex Yu",
]
__license__ = "MIT"
-__version__ = '0.15.0'
+__version__ = '0.17.0'
import base64
import calendar
@@ -161,6 +161,13 @@
"upgrade",
]
+# https://tools.ietf.org/html/rfc7231#section-8.1.3
+SAFE_METHODS = ("GET", "HEAD", "OPTIONS", "TRACE")
+
+# To change, assign to `Http().redirect_codes`
+REDIRECT_CODES = frozenset((300, 301, 302, 303, 307, 308))
+
+
from httplib2 import certs
CA_CERTS = certs.where()
@@ -315,7 +322,7 @@
# Whether to use a strict mode to parse WWW-Authenticate headers
# Might lead to bad results in case of ill-formed header value,
# so disabled by default, falling back to relaxed parsing.
-# Set to true to turn on, usefull for testing servers.
+# Set to true to turn on, useful for testing servers.
USE_WWW_AUTH_STRICT_PARSING = 0
# In regex below:
@@ -1004,10 +1011,10 @@
proxy_headers: Additional or modified headers for the proxy connect
request.
"""
- if isinstance(proxy_user, str):
- proxy_user = proxy_user.encode()
- if isinstance(proxy_pass, str):
- proxy_pass = proxy_pass.encode()
+ if isinstance(proxy_user, bytes):
+ proxy_user = proxy_user.decode()
+ if isinstance(proxy_pass, bytes):
+ proxy_pass = proxy_pass.decode()
self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass, self.proxy_headers = (
proxy_type,
proxy_host,
@@ -1467,10 +1474,14 @@
# If set to False then no redirects are followed, even safe ones.
self.follow_redirects = True
+ self.redirect_codes = REDIRECT_CODES
+
# Which HTTP methods do we apply optimistic concurrency to, i.e.
# which methods get an "if-match:" etag header added to them.
self.optimistic_concurrency_methods = ["PUT", "PATCH"]
+ self.safe_methods = list(SAFE_METHODS)
+
# If 'follow_redirects' is True, and this is set to True then
# all redirecs are followed, including unsafe ones.
self.follow_all_redirects = False
@@ -1663,10 +1674,10 @@
if (
self.follow_all_redirects
- or (method in ["GET", "HEAD"])
- or response.status == 303
+ or method in self.safe_methods
+ or response.status in (303, 308)
):
- if self.follow_redirects and response.status in [300, 301, 302, 303, 307]:
+ if self.follow_redirects and response.status in self.redirect_codes:
# Pick out the location header and basically start from the beginning
# remembering first to strip the ETag header and decrement our 'depth'
if redirections:
@@ -1686,7 +1697,7 @@
response["location"] = urllib.parse.urljoin(
absolute_uri, location
)
- if response.status == 301 and method in ["GET", "HEAD"]:
+ if response.status == 308 or (response.status == 301 and (method in self.safe_methods)):
response["-x-permanent-redirect-url"] = response["location"]
if "content-location" not in response:
response["content-location"] = absolute_uri
@@ -1723,7 +1734,7 @@
response,
content,
)
- elif response.status in [200, 203] and method in ["GET", "HEAD"]:
+ elif response.status in [200, 203] and method in self.safe_methods:
# Don't cache 206's since we aren't going to handle byte range requests
if "content-location" not in response:
response["content-location"] = absolute_uri
@@ -1822,6 +1833,7 @@
headers["accept-encoding"] = "gzip, deflate"
info = email.message.Message()
+ cachekey = None
cached_value = None
if self.cache:
cachekey = defrag_uri
@@ -1839,8 +1851,6 @@
self.cache.delete(cachekey)
cachekey = None
cached_value = None
- else:
- cachekey = None
if (
method in self.optimistic_concurrency_methods
@@ -1852,13 +1862,15 @@
# http://www.w3.org/1999/04/Editing/
headers["if-match"] = info["etag"]
- if method not in ["GET", "HEAD"] and self.cache and cachekey:
- # RFC 2616 Section 13.10
+ # https://tools.ietf.org/html/rfc7234
+ # A cache MUST invalidate the effective Request URI as well as [...] Location and Content-Location
+ # when a non-error status code is received in response to an unsafe request method.
+ if self.cache and cachekey and method not in self.safe_methods:
self.cache.delete(cachekey)
# Check the vary header in the cache to see if this request
# matches what varies in the cache.
- if method in ["GET", "HEAD"] and "vary" in info:
+ if method in self.safe_methods and "vary" in info:
vary = info["vary"]
vary_headers = vary.lower().replace(" ", "").split(",")
for header in vary_headers:
@@ -1869,11 +1881,14 @@
break
if (
- cached_value
- and method in ["GET", "HEAD"]
- and self.cache
+ self.cache
+ and cached_value
+ and (method in self.safe_methods or info["status"] == "308")
and "range" not in headers
):
+ redirect_method = method
+ if info["status"] not in ("307", "308"):
+ redirect_method = "GET"
if "-x-permanent-redirect-url" in info:
# Should cached permanent redirects be counted in our redirection count? For now, yes.
if redirections <= 0:
@@ -1884,7 +1899,7 @@
)
(response, new_content) = self.request(
info["-x-permanent-redirect-url"],
- method="GET",
+ method=redirect_method,
headers=headers,
redirections=redirections - 1,
)
diff --git a/python3/httplib2/socks.py b/python3/httplib2/socks.py
index 2926b4e..cc68e63 100644
--- a/python3/httplib2/socks.py
+++ b/python3/httplib2/socks.py
@@ -238,7 +238,15 @@
headers - Additional or modified headers for the proxy connect
request.
"""
- self.__proxy = (proxytype, addr, port, rdns, username, password, headers)
+ self.__proxy = (
+ proxytype,
+ addr,
+ port,
+ rdns,
+ username.encode() if username else None,
+ password.encode() if password else None,
+ headers,
+ )
def __negotiatesocks5(self, destaddr, destport):
"""__negotiatesocks5(self,destaddr,destport)
diff --git a/setup.py b/setup.py
index 33c8827..a3be8d4 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@
import sys
pkgdir = {"": "python%s" % sys.version_info[0]}
-VERSION = '0.15.0'
+VERSION = '0.17.0'
# `python setup.py test` uses existing Python environment, no virtualenv, no pip.
diff --git a/tests/__init__.py b/tests/__init__.py
index 496652b..a15db9e 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -406,10 +406,12 @@
request = HttpRequest.from_buffered(buf)
if request is None:
break
+ # print("--- debug request\n" + request.raw.decode("ascii", "replace"))
i += 1
request.client_sock = sock
request.number = i
response = request_handler(request=request)
+ # print("--- debug response\n" + response.decode("ascii", "replace"))
sock.sendall(response)
request.client_sock = None
if not tick(request):
diff --git a/tests/test_http.py b/tests/test_http.py
index 97b52dc..df99016 100644
--- a/tests/test_http.py
+++ b/tests/test_http.py
@@ -622,6 +622,57 @@
assert content == b"final content\n"
+def test_post_307():
+ # 307: follow with same method
+ http = httplib2.Http(cache=tests.get_cache_path(), timeout=1)
+ http.follow_all_redirects = True
+ r307 = tests.http_response_bytes(status=307, headers={"location": "/final"})
+ r200 = tests.http_response_bytes(status=200, body=b"final content\n")
+
+ with tests.server_list_http([r307, r200, r307, r200]) as uri:
+ response, content = http.request(uri, "POST")
+ assert response.previous.status == 307
+ assert not response.previous.fromcache
+ assert response.status == 200
+ assert not response.fromcache
+ assert content == b"final content\n"
+
+ response, content = http.request(uri, "POST")
+ assert response.previous.status == 307
+ assert not response.previous.fromcache
+ assert response.status == 200
+ assert not response.fromcache
+ assert content == b"final content\n"
+
+
+def test_change_308():
+ # 308: follow with same method, cache redirect
+ http = httplib2.Http(cache=tests.get_cache_path(), timeout=1)
+ routes = {
+ "/final": tests.make_http_reflect(),
+ "": tests.http_response_bytes(
+ status="308 Permanent Redirect",
+ add_date=True,
+ headers={"cache-control": "max-age=300", "location": "/final"},
+ ),
+ }
+
+ with tests.server_route(routes, request_count=3) as uri:
+ response, content = http.request(uri, "CHANGE", body=b"hello308")
+ assert response.previous.status == 308
+ assert not response.previous.fromcache
+ assert response.status == 200
+ assert not response.fromcache
+ assert content.startswith(b"CHANGE /final HTTP")
+
+ response, content = http.request(uri, "CHANGE")
+ assert response.previous.status == 308
+ assert response.previous.fromcache
+ assert response.status == 200
+ assert not response.fromcache
+ assert content.startswith(b"CHANGE /final HTTP")
+
+
def test_get_410():
# Test that we pass 410's through
http = httplib2.Http()
@@ -643,3 +694,12 @@
assert response.status == 200
assert content == b"content"
assert response["link"], "link1, link2"
+
+
+def test_custom_redirect_codes():
+ http = httplib2.Http()
+ http.redirect_codes = set([300])
+ with tests.server_const_http(status=301, request_count=1) as uri:
+ response, content = http.request(uri, "GET")
+ assert response.status == 301
+ assert response.previous is None
diff --git a/tests/test_proxy.py b/tests/test_proxy.py
index 375367f..4ec8aea 100644
--- a/tests/test_proxy.py
+++ b/tests/test_proxy.py
@@ -32,8 +32,8 @@
pi = httplib2.proxy_info_from_url("http://zoidberg:fish@someproxy:99")
assert pi.proxy_host == "someproxy"
assert pi.proxy_port == 99
- assert pi.proxy_user == b"zoidberg"
- assert pi.proxy_pass == b"fish"
+ assert pi.proxy_user == "zoidberg"
+ assert pi.proxy_pass == "fish"
def test_from_env():
