diff --git a/test/with_dummyserver/async/test_poolmanager.py b/test/with_dummyserver/async/test_poolmanager.py index 63cba1ca..f30f94ca 100644 --- a/test/with_dummyserver/async/test_poolmanager.py +++ b/test/with_dummyserver/async/test_poolmanager.py @@ -1,11 +1,15 @@ import io +import json import pytest +from ahip.base import DEFAULT_PORTS from ahip import PoolManager, Retry -from ahip.exceptions import UnrewindableBodyError +from ahip.exceptions import MaxRetryError, UnrewindableBodyError from test.with_dummyserver import conftest from dummyserver.testcase import HTTPDummyServerTestCase +from test import LONG_TIMEOUT + class TestPoolManager(HTTPDummyServerTestCase): @classmethod @@ -34,6 +38,340 @@ async def test_redirect(self, backend, anyio_backend): assert r.status == 200 assert r.data == b"Dummy server!" + @conftest.test_all_backends + async def test_redirect_twice(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "%s/redirect" % self.base_url}, + redirect=False, + ) + + assert r.status == 303 + + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={ + "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) + }, + ) + + assert r.status == 200 + assert r.data == b"Dummy server!" + + @conftest.test_all_backends + async def test_redirect_to_relative_url(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "/redirect"}, + redirect=False, + ) + + assert r.status == 303 + + r = await http.request( + "GET", "%s/redirect" % self.base_url, fields={"target": "/redirect"} + ) + + assert r.status == 200 + assert r.data == b"Dummy server!" + + @conftest.test_all_backends + async def test_cross_host_redirect(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + cross_host_location = "%s/echo?a=b" % self.base_url_alt + with pytest.raises(MaxRetryError): + await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": cross_host_location}, + timeout=LONG_TIMEOUT, + retries=0, + ) + + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "%s/echo?a=b" % self.base_url_alt}, + timeout=LONG_TIMEOUT, + retries=1, + ) + + assert r._pool.host == self.host_alt + + @conftest.test_all_backends + async def test_too_many_redirects(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + with pytest.raises(MaxRetryError): + await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={ + "target": "%s/redirect?target=%s/" + % (self.base_url, self.base_url) + }, + retries=1, + ) + + with pytest.raises(MaxRetryError): + await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={ + "target": "%s/redirect?target=%s/" + % (self.base_url, self.base_url) + }, + retries=Retry(total=None, redirect=1), + ) + + @conftest.test_all_backends + async def test_redirect_cross_host_remove_headers(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "%s/headers" % self.base_url_alt}, + headers={"Authorization": "foo"}, + ) + + assert r.status == 200 + + data = json.loads(r.data.decode("utf-8")) + + assert "Authorization" not in data + + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "%s/headers" % self.base_url_alt}, + headers={"authorization": "foo"}, + ) + + assert r.status == 200 + + data = json.loads(r.data.decode("utf-8")) + + assert "authorization" not in data + assert "Authorization" not in data + + @conftest.test_all_backends + async def test_redirect_cross_host_no_remove_headers(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "%s/headers" % self.base_url_alt}, + headers={"Authorization": "foo"}, + retries=Retry(remove_headers_on_redirect=[]), + ) + + assert r.status == 200 + + data = json.loads(r.data.decode("utf-8")) + + assert data["Authorization"] == "foo" + + @conftest.test_all_backends + async def test_redirect_cross_host_set_removed_headers( + self, backend, anyio_backend + ): + with PoolManager(backend=backend) as http: + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "%s/headers" % self.base_url_alt}, + headers={"X-API-Secret": "foo", "Authorization": "bar"}, + retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), + ) + + assert r.status == 200 + + data = json.loads(r.data.decode("utf-8")) + + assert "X-API-Secret" not in data + assert data["Authorization"] == "bar" + + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={"target": "%s/headers" % self.base_url_alt}, + headers={"x-api-secret": "foo", "authorization": "bar"}, + retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), + ) + + assert r.status == 200 + + data = json.loads(r.data.decode("utf-8")) + + assert "x-api-secret" not in data + assert "X-API-Secret" not in data + assert data["Authorization"] == "bar" + + @conftest.test_all_backends + async def test_raise_on_redirect(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + r = await http.request( + "GET", + "%s/redirect" % self.base_url, + fields={ + "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) + }, + retries=Retry(total=None, redirect=1, raise_on_redirect=False), + ) + + assert r.status == 303 + + @conftest.test_all_backends + async def test_raise_on_status(self, backend, anyio_backend): + with PoolManager(backend=backend) as http: + with pytest.raises(MaxRetryError): + # the default is to raise + r = await http.request( + "GET", + "%s/status" % self.base_url, + fields={"status": "500 Internal Server Error"}, + retries=Retry(total=1, status_forcelist=range(500, 600)), + ) + + with pytest.raises(MaxRetryError): + # raise explicitly + r = await http.request( + "GET", + "%s/status" % self.base_url, + fields={"status": "500 Internal Server Error"}, + retries=Retry( + total=1, status_forcelist=range(500, 600), raise_on_status=True + ), + ) + + # don't raise + r = await http.request( + "GET", + "%s/status" % self.base_url, + fields={"status": "500 Internal Server Error"}, + retries=Retry( + total=1, status_forcelist=range(500, 600), raise_on_status=False + ), + ) + + assert r.status == 500 + + @pytest.mark.parametrize( + ["target", "expected_target"], + [ + ("/echo_uri?q=1#fragment", b"/echo_uri?q=1"), + ("/echo_uri?#", b"/echo_uri?"), + ("/echo_uri#?", b"/echo_uri"), + ("/echo_uri#?#", b"/echo_uri"), + ("/echo_uri??#", b"/echo_uri??"), + ("/echo_uri?%3f#", b"/echo_uri?%3F"), + ("/echo_uri?%3F#", b"/echo_uri?%3F"), + ("/echo_uri?[]", b"/echo_uri?%5B%5D"), + ], + ) + @conftest.test_all_backends + async def test_encode_http_target( + self, target, expected_target, backend, anyio_backend + ): + with PoolManager() as http: + url = "http://%s:%d%s" % (self.host, self.port, target) + r = await http.request("GET", url) + assert r.data == expected_target + + @conftest.test_all_backends + async def test_missing_port(self, backend, anyio_backend): + # Can a URL that lacks an explicit port like ':80' succeed, or + # will all such URLs fail with an error? + + with PoolManager(backend=backend) as http: + # By globally adjusting `DEFAULT_PORTS` we pretend for a moment + # that HTTP's default port is not 80, but is the port at which + # our test server happens to be listening. + DEFAULT_PORTS["http"] = self.port + try: + r = await http.request("GET", "http://%s/" % self.host, retries=0) + finally: + DEFAULT_PORTS["http"] = 80 + + assert r.status == 200 + assert r.data == b"Dummy server!" + + @conftest.test_all_backends + async def test_headers(self, backend, anyio_backend): + with PoolManager(backend=backend, headers={"Foo": "bar"}) as http: + r = await http.request("GET", "%s/headers" % self.base_url) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") == "bar" + + r = await http.request("POST", "%s/headers" % self.base_url) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") == "bar" + + r = await http.request_encode_url("GET", "%s/headers" % self.base_url) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") == "bar" + + r = await http.request_encode_body("POST", "%s/headers" % self.base_url) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") == "bar" + + r = await http.request_encode_url( + "GET", "%s/headers" % self.base_url, headers={"Baz": "quux"} + ) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") is None + assert returned_headers.get("Baz") == "quux" + + r = await http.request_encode_body( + "GET", "%s/headers" % self.base_url, headers={"Baz": "quux"} + ) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") is None + assert returned_headers.get("Baz") == "quux" + + @conftest.test_all_backends + async def test_http_with_ssl_keywords(self, backend, anyio_backend): + with PoolManager(backend=backend, ca_certs="REQUIRED") as http: + r = await http.request("GET", "http://%s:%s/" % (self.host, self.port)) + assert r.status == 200 + + @conftest.test_all_backends + async def test_http_with_ca_cert_dir(self, backend, anyio_backend): + with PoolManager( + backend=backend, ca_certs="REQUIRED", ca_cert_dir="/nosuchdir" + ) as http: + r = await http.request("GET", "http://%s:%s/" % (self.host, self.port)) + assert r.status == 200 + + @conftest.test_all_backends + async def test_cleanup_on_connection_error(self, backend, anyio_backend): + """ + Test that connections are recycled to the pool on + connection errors where no http response is received. + """ + poolsize = 3 + + with PoolManager(backend=backend, maxsize=poolsize, block=True) as http: + pool = http.connection_from_host(self.host, self.port) + assert pool.pool.qsize() == poolsize + + # force a connection error by supplying a non-existent + # url. We won't get a response for this and so the + # conn won't be implicitly returned to the pool. + url = "%s/redirect" % self.base_url + with pytest.raises(MaxRetryError): + await http.request("GET", url, fields={"target": "/"}, retries=0) + + r = await http.request("GET", url, fields={"target": "/"}, retries=1) + r.release_conn() + + # the pool should still contain poolsize elements + assert pool.pool.qsize() == poolsize + class TestFileUploads(HTTPDummyServerTestCase): @classmethod @@ -101,7 +439,7 @@ def tell(self): # which is unsupported by BytesIO. headers = {"Content-Length": "8"} - with PoolManager() as http: + with PoolManager(backend=backend) as http: with pytest.raises(UnrewindableBodyError) as e: await http.urlopen("PUT", url, headers=headers, body=body) assert "Unable to record file position for" in str(e.value) @@ -120,7 +458,7 @@ def seek(self, *_): # which is unsupported by BytesIO. headers = {"Content-Length": "8"} - with PoolManager() as http: + with PoolManager(backend=backend) as http: with pytest.raises(UnrewindableBodyError) as e: await http.urlopen("PUT", url, headers=headers, body=body) assert ( diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 98a621ac..d7cdd256 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -1,362 +1,17 @@ -import json import time import pytest from dummyserver.server import HAS_IPV6 from dummyserver.testcase import HTTPDummyServerTestCase, IPv6HTTPDummyServerTestCase -from hip.base import DEFAULT_PORTS from hip.poolmanager import PoolManager from hip.exceptions import MaxRetryError, NewConnectionError from hip.util.retry import Retry, RequestHistory -from test import LONG_TIMEOUT - # Retry failed tests pytestmark = pytest.mark.flaky -class TestPoolManager(HTTPDummyServerTestCase): - @classmethod - def setup_class(cls): - super(TestPoolManager, cls).setup_class() - cls.base_url = "http://%s:%d" % (cls.host, cls.port) - cls.base_url_alt = "http://%s:%d" % (cls.host_alt, cls.port) - - def test_redirect(self): - with PoolManager() as http: - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/" % self.base_url}, - redirect=False, - ) - - assert r.status == 303 - - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/" % self.base_url}, - ) - - assert r.status == 200 - assert r.data == b"Dummy server!" - - def test_redirect_twice(self): - with PoolManager() as http: - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/redirect" % self.base_url}, - redirect=False, - ) - - assert r.status == 303 - - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={ - "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) - }, - ) - - assert r.status == 200 - assert r.data == b"Dummy server!" - - def test_redirect_to_relative_url(self): - with PoolManager() as http: - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "/redirect"}, - redirect=False, - ) - - assert r.status == 303 - - r = http.request( - "GET", "%s/redirect" % self.base_url, fields={"target": "/redirect"} - ) - - assert r.status == 200 - assert r.data == b"Dummy server!" - - def test_cross_host_redirect(self): - with PoolManager() as http: - cross_host_location = "%s/echo?a=b" % self.base_url_alt - with pytest.raises(MaxRetryError): - http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": cross_host_location}, - timeout=LONG_TIMEOUT, - retries=0, - ) - - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/echo?a=b" % self.base_url_alt}, - timeout=LONG_TIMEOUT, - retries=1, - ) - - assert r._pool.host == self.host_alt - - def test_too_many_redirects(self): - with PoolManager() as http: - with pytest.raises(MaxRetryError): - http.request( - "GET", - "%s/redirect" % self.base_url, - fields={ - "target": "%s/redirect?target=%s/" - % (self.base_url, self.base_url) - }, - retries=1, - ) - - with pytest.raises(MaxRetryError): - http.request( - "GET", - "%s/redirect" % self.base_url, - fields={ - "target": "%s/redirect?target=%s/" - % (self.base_url, self.base_url) - }, - retries=Retry(total=None, redirect=1), - ) - - def test_redirect_cross_host_remove_headers(self): - with PoolManager() as http: - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"Authorization": "foo"}, - ) - - assert r.status == 200 - - data = json.loads(r.data.decode("utf-8")) - - assert "Authorization" not in data - - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"authorization": "foo"}, - ) - - assert r.status == 200 - - data = json.loads(r.data.decode("utf-8")) - - assert "authorization" not in data - assert "Authorization" not in data - - def test_redirect_cross_host_no_remove_headers(self): - with PoolManager() as http: - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"Authorization": "foo"}, - retries=Retry(remove_headers_on_redirect=[]), - ) - - assert r.status == 200 - - data = json.loads(r.data.decode("utf-8")) - - assert data["Authorization"] == "foo" - - def test_redirect_cross_host_set_removed_headers(self): - with PoolManager() as http: - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"X-API-Secret": "foo", "Authorization": "bar"}, - retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), - ) - - assert r.status == 200 - - data = json.loads(r.data.decode("utf-8")) - - assert "X-API-Secret" not in data - assert data["Authorization"] == "bar" - - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={"target": "%s/headers" % self.base_url_alt}, - headers={"x-api-secret": "foo", "authorization": "bar"}, - retries=Retry(remove_headers_on_redirect=["X-API-Secret"]), - ) - - assert r.status == 200 - - data = json.loads(r.data.decode("utf-8")) - - assert "x-api-secret" not in data - assert "X-API-Secret" not in data - assert data["Authorization"] == "bar" - - def test_raise_on_redirect(self): - with PoolManager() as http: - r = http.request( - "GET", - "%s/redirect" % self.base_url, - fields={ - "target": "%s/redirect?target=%s/" % (self.base_url, self.base_url) - }, - retries=Retry(total=None, redirect=1, raise_on_redirect=False), - ) - - assert r.status == 303 - - def test_raise_on_status(self): - with PoolManager() as http: - with pytest.raises(MaxRetryError): - # the default is to raise - r = http.request( - "GET", - "%s/status" % self.base_url, - fields={"status": "500 Internal Server Error"}, - retries=Retry(total=1, status_forcelist=range(500, 600)), - ) - - with pytest.raises(MaxRetryError): - # raise explicitly - r = http.request( - "GET", - "%s/status" % self.base_url, - fields={"status": "500 Internal Server Error"}, - retries=Retry( - total=1, status_forcelist=range(500, 600), raise_on_status=True - ), - ) - - # don't raise - r = http.request( - "GET", - "%s/status" % self.base_url, - fields={"status": "500 Internal Server Error"}, - retries=Retry( - total=1, status_forcelist=range(500, 600), raise_on_status=False - ), - ) - - assert r.status == 500 - - @pytest.mark.parametrize( - ["target", "expected_target"], - [ - ("/echo_uri?q=1#fragment", b"/echo_uri?q=1"), - ("/echo_uri?#", b"/echo_uri?"), - ("/echo_uri#?", b"/echo_uri"), - ("/echo_uri#?#", b"/echo_uri"), - ("/echo_uri??#", b"/echo_uri??"), - ("/echo_uri?%3f#", b"/echo_uri?%3F"), - ("/echo_uri?%3F#", b"/echo_uri?%3F"), - ("/echo_uri?[]", b"/echo_uri?%5B%5D"), - ], - ) - def test_encode_http_target(self, target, expected_target): - with PoolManager() as http: - url = "http://%s:%d%s" % (self.host, self.port, target) - r = http.request("GET", url) - assert r.data == expected_target - - def test_missing_port(self): - # Can a URL that lacks an explicit port like ':80' succeed, or - # will all such URLs fail with an error? - - with PoolManager() as http: - # By globally adjusting `DEFAULT_PORTS` we pretend for a moment - # that HTTP's default port is not 80, but is the port at which - # our test server happens to be listening. - DEFAULT_PORTS["http"] = self.port - try: - r = http.request("GET", "http://%s/" % self.host, retries=0) - finally: - DEFAULT_PORTS["http"] = 80 - - assert r.status == 200 - assert r.data == b"Dummy server!" - - def test_headers(self): - with PoolManager(headers={"Foo": "bar"}) as http: - r = http.request("GET", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) - assert returned_headers.get("Foo") == "bar" - - r = http.request("POST", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) - assert returned_headers.get("Foo") == "bar" - - r = http.request_encode_url("GET", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) - assert returned_headers.get("Foo") == "bar" - - r = http.request_encode_body("POST", "%s/headers" % self.base_url) - returned_headers = json.loads(r.data.decode()) - assert returned_headers.get("Foo") == "bar" - - r = http.request_encode_url( - "GET", "%s/headers" % self.base_url, headers={"Baz": "quux"} - ) - returned_headers = json.loads(r.data.decode()) - assert returned_headers.get("Foo") is None - assert returned_headers.get("Baz") == "quux" - - r = http.request_encode_body( - "GET", "%s/headers" % self.base_url, headers={"Baz": "quux"} - ) - returned_headers = json.loads(r.data.decode()) - assert returned_headers.get("Foo") is None - assert returned_headers.get("Baz") == "quux" - - def test_http_with_ssl_keywords(self): - with PoolManager(ca_certs="REQUIRED") as http: - r = http.request("GET", "http://%s:%s/" % (self.host, self.port)) - assert r.status == 200 - - def test_http_with_ca_cert_dir(self): - with PoolManager(ca_certs="REQUIRED", ca_cert_dir="/nosuchdir") as http: - r = http.request("GET", "http://%s:%s/" % (self.host, self.port)) - assert r.status == 200 - - def test_cleanup_on_connection_error(self): - """ - Test that connections are recycled to the pool on - connection errors where no http response is received. - """ - poolsize = 3 - - with PoolManager(maxsize=poolsize, block=True) as http: - pool = http.connection_from_host(self.host, self.port) - assert pool.pool.qsize() == poolsize - - # force a connection error by supplying a non-existent - # url. We won't get a response for this and so the - # conn won't be implicitly returned to the pool. - url = "%s/redirect" % self.base_url - with pytest.raises(MaxRetryError): - http.request("GET", url, fields={"target": "/"}, retries=0) - - r = http.request("GET", url, fields={"target": "/"}, retries=1) - r.release_conn() - - # the pool should still contain poolsize elements - assert pool.pool.qsize() == poolsize - - class TestRetry(HTTPDummyServerTestCase): @classmethod def setup_class(self):