diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index ae70a6b32b..7446185cd2 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -22,6 +22,23 @@ jobs: with: python-version: "${{ matrix.python-version }}" allow-prereleases: true + - uses: pyodide/pyodide-actions/install-browser@v2 + if: ${{ matrix.python-version ==3.12 }} + with: + runner: selenium + browser: chrome + browser-version: 120 + - uses: pyodide/pyodide-actions/install-browser@v2 + if: ${{ matrix.python-version ==3.12 }} + with: + runner: selenium + browser: node + browser-version: 22 + - uses: pyodide/pyodide-actions/download-pyodide@v2 + if: ${{ matrix.python-version ==3.12 }} + with: + version: 0.26.2 + to: pyodide_dist - name: "Install dependencies" run: "scripts/install" - name: "Run linting checks" diff --git a/.gitignore b/.gitignore index 49e14ccc8d..f6a8e8d530 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ venv*/ .python-version build/ dist/ +pyodide_dist \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ee0f81f9c3..ea349d17bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] + +* Support for Emscripten (#3330) + ## 0.28.1 (6th December, 2024) * Fix SSL case where `verify=False` together with client side certificates. diff --git a/docs/advanced/emscripten.md b/docs/advanced/emscripten.md new file mode 100644 index 0000000000..c59d2a0ee3 --- /dev/null +++ b/docs/advanced/emscripten.md @@ -0,0 +1,57 @@ +--- +template: pyodide.html +--- +# Emscripten Support + +httpx has support for running on Webassembly / Emscripten using [pyodide](https://github.com/pyodide/pyodide/). + +In Emscripten, all network connections are handled by the enclosing Javascript runtime. As such, there is limited control over various features. In particular: + +- Proxy servers are handled by the runtime, so you cannot control them. +- httpx has no control over connection pooling. +- Certificate handling is done by the browser, so you cannot modify it. +- Requests are constrained by cross-origin isolation settings in the same way as any request that is originated by Javascript code. +- On browsers, timeouts will not work in the main browser thread unless your browser supports [Javascript Promise Integration](https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md). This is currently behind a flag on chrome, and not yet supported by non-chromium browsers. +- On node.js, synchronous requests will only work if you enable Javascript Promise Integration. You can do this using the `--experimental-wasm-stack-switching` flag when you run the node executable. + +## Try it in your browser + +Use the following live example to test httpx in your web browser. You can change the code below and hit run again to test different features or web addresses. + +
+ +
+
+ +
+import httpx +print("Sending response using httpx in the browser:") +print("--------------------------------------------") +r=httpx.get("http://www.example.com") +print("Status = ",r.status_code) +print("Response = ",r.text[:50],"...") +
+ + +## Build it +Because this is a pure python module, building is the same as ever (`python -m build`), or use the built wheel from pypi. + +## Testing Custom Builds of httpx in Emscripten +Once you have a wheel you can test it in your browser. You can do this using the [pyodide console]( +https://pyodide.org/en/latest/console.html), or by hosting your own web page. + +1) To test in pyodide console, serve the wheel file via http (e.g. by calling python -m `http.server` in the dist directory.) Then in the [pyodide console]( +https://pyodide.org/en/latest/console.html), type the following, replacing the URL of the locally served wheel. + +``` +import pyodide_js as pjs +import ssl,certifi,idna +pjs.loadPackage("") +import httpx +# Now httpx should work +``` + +2) To test a custom built wheel in your own web page, create a page which loads the pyodide javascript (see the [instructions](https://pyodide.org/en/stable/usage/index.html) on the pyodide website), then call `pyodide.loadPackage` on your pyodide instance, pointing it at the wheel file. Then make sure you load the dependencies by loading the ssl,certifi and idna packages (which are part of pyodide - call `pyodide.loadPackage` for each one and pass it just the package name.) + +3) To test in node.js, make sure you have a pyodide distribution downloaded to a known folder, then load pyodide following the instructions on the pyodide website (https://pyodide.org/en/stable/usage/index.html). You can then call await `pyodide.loadPackage('');` and httpx should be available as a package in pyodide. You need at version 0.26.2 or later of pyodide. + diff --git a/docs/index.md b/docs/index.md index c2210bc74f..5774d336c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -148,3 +148,6 @@ $ pip install httpx[brotli,zstd] HTTPX requires Python 3.8+ [sync-support]: https://github.com/encode/httpx/issues/572 + +## Pyodide / Emscripten Support +There is experimental support for running in Webassembly under the pyodide runtime. See the [Emscripten](advanced/emscripten.md) page for more details. diff --git a/docs/overrides/pyodide.html b/docs/overrides/pyodide.html new file mode 100644 index 0000000000..dcb14d75e5 --- /dev/null +++ b/docs/overrides/pyodide.html @@ -0,0 +1,128 @@ +{% extends "main.html" %} +{% block styles %} + {{ super() }} + + +{% endblock %} + +{% block scripts %} +{{ super() }} + + + +{% endblock %} + + + + diff --git a/httpx/_api.py b/httpx/_api.py index c3cda1ecda..282834d7a8 100644 --- a/httpx/_api.py +++ b/httpx/_api.py @@ -3,6 +3,9 @@ import typing from contextlib import contextmanager +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + from ._client import Client from ._config import DEFAULT_TIMEOUT_CONFIG from ._models import Response diff --git a/httpx/_client.py b/httpx/_client.py index 2249231f8c..30128a9de7 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -28,8 +28,12 @@ ) from ._models import Cookies, Headers, Request, Response from ._status_codes import codes -from ._transports.base import AsyncBaseTransport, BaseTransport -from ._transports.default import AsyncHTTPTransport, HTTPTransport +from ._transports import ( + AsyncBaseTransport, + AsyncHTTPTransport, + BaseTransport, + HTTPTransport, +) from ._types import ( AsyncByteStream, AuthTypes, @@ -51,6 +55,7 @@ if typing.TYPE_CHECKING: import ssl # pragma: no cover + __all__ = ["USE_CLIENT_DEFAULT", "AsyncClient", "Client"] # The type annotation for @classmethod and context managers here follows PEP 484 diff --git a/httpx/_transports/__init__.py b/httpx/_transports/__init__.py index 7a321053b2..2669c30d70 100644 --- a/httpx/_transports/__init__.py +++ b/httpx/_transports/__init__.py @@ -1,9 +1,24 @@ +import sys + from .asgi import * from .base import * -from .default import * from .mock import * from .wsgi import * +if sys.platform == "emscripten": # pragma: nocover + # in emscripten we use javascript fetch + from .jsfetch import * + + # override default transport names + HTTPTransport = JavascriptFetchTransport + AsyncHTTPTransport = AsyncJavascriptFetchTransport +else: + # everywhere else we use httpcore + from .httpcore import * + + HTTPTransport = HTTPCoreTransport + AsyncHTTPTransport = AsyncHTTPCoreTransport + __all__ = [ "ASGITransport", "AsyncBaseTransport", diff --git a/httpx/_transports/default.py b/httpx/_transports/httpcore.py similarity index 96% rename from httpx/_transports/default.py rename to httpx/_transports/httpcore.py index d5aa05ff23..9d8d52f55e 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/httpcore.py @@ -11,16 +11,16 @@ # Disable HTTP/2 on a single specific domain. mounts = { - "all://": httpx.HTTPTransport(http2=True), - "all://*example.org": httpx.HTTPTransport() + "all://": httpx.HTTPCoreTransport(http2=True), + "all://*example.org": httpx.HTTPCoreTransport() } # Using advanced httpcore configuration, with connection retries. -transport = httpx.HTTPTransport(retries=1) +transport = httpx.HTTPCoreTransport(retries=1) client = httpx.Client(transport=transport) # Using advanced httpcore configuration, with unix domain sockets. -transport = httpx.HTTPTransport(uds="socket.uds") +transport = httpx.HTTPCoreTransport(uds="socket.uds") client = httpx.Client(transport=transport) """ @@ -57,8 +57,8 @@ from .._urls import URL from .base import AsyncBaseTransport, BaseTransport -T = typing.TypeVar("T", bound="HTTPTransport") -A = typing.TypeVar("A", bound="AsyncHTTPTransport") +T = typing.TypeVar("T", bound="HTTPCoreTransport") +A = typing.TypeVar("A", bound="AsyncHTTPCoreTransport") SOCKET_OPTION = typing.Union[ typing.Tuple[int, int, int], @@ -66,7 +66,7 @@ typing.Tuple[int, int, None, int], ] -__all__ = ["AsyncHTTPTransport", "HTTPTransport"] +__all__ = ["AsyncHTTPCoreTransport", "HTTPCoreTransport"] HTTPCORE_EXC_MAP: dict[type[Exception], type[httpx.HTTPError]] = {} @@ -132,7 +132,7 @@ def close(self) -> None: self._httpcore_stream.close() -class HTTPTransport(BaseTransport): +class HTTPCoreTransport(BaseTransport): def __init__( self, verify: ssl.SSLContext | str | bool = True, @@ -276,7 +276,7 @@ async def aclose(self) -> None: await self._httpcore_stream.aclose() -class AsyncHTTPTransport(AsyncBaseTransport): +class AsyncHTTPCoreTransport(AsyncBaseTransport): def __init__( self, verify: ssl.SSLContext | str | bool = True, diff --git a/httpx/_transports/jsfetch.py b/httpx/_transports/jsfetch.py new file mode 100644 index 0000000000..8da3c9ddaa --- /dev/null +++ b/httpx/_transports/jsfetch.py @@ -0,0 +1,467 @@ +""" +Custom transport for Pyodide on Emscripten. + +In sync mode it relies on use of the Javascript Promise Integration +feature which is currently experimental in webassembly and only works +in some places. Specifically in chrome JSPI is behind either a flag or an +origin trial, in node 20 or newer you need the --experimental-wasm-stack-switching +flag. Firefox is not currently supported. + +See https://github.com/WebAssembly/js-promise-integration/ + +In async mode it uses the standard fetch api, which should work +anywhere that pyodide works. +""" + +from __future__ import annotations + +import email.parser +import typing +from types import TracebackType + +import js +import pyodide + +if typing.TYPE_CHECKING: + import ssl # pragma: nocover + +from .._config import DEFAULT_LIMITS, Limits +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ReadError, + ReadTimeout, + RequestError, +) +from .._models import Request, Response +from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream +from .base import AsyncBaseTransport, BaseTransport + +T = typing.TypeVar("T", bound="JavascriptFetchTransport") +A = typing.TypeVar("A", bound="AsyncJavascriptFetchTransport") + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + +__all__ = ["AsyncJavascriptFetchTransport", "JavascriptFetchTransport"] + +""" +There are some headers that trigger unintended CORS preflight requests. +See also https://github.com/koenvo/pyodide-http/issues/22 +""" +HEADERS_TO_IGNORE = ("user-agent",) + + +def _run_sync_with_timeout( + promise: typing.Awaitable[pyodide.ffi.JsProxy], + timeout: float, + abort_controller_js: pyodide.ffi.JsProxy, + TimeoutExceptionType: type[RequestError], + ErrorExceptionType: type[RequestError], +) -> pyodide.ffi.JsProxy: + """await a javascript promise synchronously with a timeout set via the + AbortController and return the resulting javascript proxy + + Args: + promise (Awaitable): Javascript promise to await + timeout (float): Timeout in seconds + abort_controller_js (Any): A javascript AbortController object, used on timeout + TimeoutExceptionType (type[Exception]): An exception type to raise on timeout + ErrorExceptionType (type[Exception]): An exception type to raise on error + + Raises: + TimeoutExceptionType: If the request times out + ErrorExceptionType: If the request raises a Javascript exception + + Returns: + _type_: The result of awaiting the promise. + """ + timer_id = None + if timeout > 0: + timer_id = js.setTimeout( + abort_controller_js.abort.bind(abort_controller_js), int(timeout * 1000) + ) + try: + from pyodide.ffi import run_sync + + # run_sync here uses WebAssembly Javascript Promise Integration to + # suspend python until the Javascript promise resolves. + return run_sync(promise) + except pyodide.ffi.JsException as err: + if err.name == "AbortError": + raise TimeoutExceptionType(message="Request timed out") + timer_id = None + else: + raise ErrorExceptionType(message=err.message) + finally: + if timer_id is not None: + js.clearTimeout(timer_id) + + +async def _run_async_with_timeout( + promise: typing.Awaitable[pyodide.ffi.JsProxy], + timeout: float, + abort_controller_js: pyodide.ffi.JsProxy, + TimeoutExceptionType: type[RequestError], + ErrorExceptionType: type[RequestError], +) -> pyodide.ffi.JsProxy: + """await a javascript promise asynchronously with a timeout set via the + AbortController + + Args: + promise (Awaitable): Javascript promise to await + timeout (float): Timeout in seconds + abort_controller_js (Any): A javascript AbortController object, used on timeout + TimeoutExceptionType (type[Exception]): An exception type to raise on timeout + ErrorExceptionType (type[Exception]): An exception type to raise on error + + Raises: + TimeoutException: If the request times out + NetworkError: If the request raises a Javascript exception + + Returns: + _type_: The result of awaiting the promise. + """ + timer_id = None + if timeout > 0: + timer_id = js.setTimeout( + abort_controller_js.abort.bind(abort_controller_js), int(timeout * 1000) + ) + try: + return await promise + except pyodide.ffi.JsException as err: + if err.name == "AbortError": + raise TimeoutExceptionType(message="Request timed out") + timer_id = None + else: + raise ErrorExceptionType(message=err.message) + finally: + if timer_id is not None: + js.clearTimeout(timer_id) + + +class EmscriptenStream(SyncByteStream): + def __init__( + self, + response_stream_js: pyodide.ffi.JsProxy, + timeout: float, + abort_controller_js: pyodide.ffi.JsProxy, + ) -> None: + self._stream_js = response_stream_js + self.timeout = timeout + self.abort_controller_js = abort_controller_js + + def __iter__(self) -> typing.Iterator[bytes]: + while True: + result_js = _run_sync_with_timeout( + self._stream_js.read(), + self.timeout, + self.abort_controller_js, + ReadTimeout, + ReadError, + ) + if result_js.done: + return + else: + this_buffer = result_js.value.to_py() + yield this_buffer + + def close(self) -> None: + self._stream_js = None + + +class JavascriptFetchTransport(BaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + pass + + def __enter__(self: T) -> T: # Use generics for subclass support. + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + pass + + def handle_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, SyncByteStream) + if not self._can_use_jspi(): + return self._no_jspi_fallback(request) + req_body: bytes | None = b"".join(request.stream) + if req_body is not None and len(req_body) == 0: + req_body = None + conn_timeout = 0.0 + read_timeout = 0.0 + if "timeout" in request.extensions: + timeout_dict = request.extensions["timeout"] + if timeout_dict is not None: + if "connect" in timeout_dict: + conn_timeout = timeout_dict["connect"] + if "read" in timeout_dict: + read_timeout = timeout_dict["connect"] + abort_controller_js = js.AbortController.new() + headers = { + k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE + } + fetch_data = { + "headers": headers, + "body": pyodide.ffi.to_js(req_body), + "method": request.method, + "signal": abort_controller_js.signal, + } + + fetcher_promise_js = js.fetch( + request.url, + pyodide.ffi.to_js(fetch_data, dict_converter=js.Object.fromEntries), + ) + + response_js = _run_sync_with_timeout( + fetcher_promise_js, + conn_timeout, + abort_controller_js, + ConnectTimeout, + ConnectError, + ) + + headers = {} + header_iter = response_js.headers.entries() + while True: + iter_value_js = header_iter.next() + if getattr(iter_value_js, "done", False): + break + else: + headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1]) + # fix content-encoding headers because the javascript fetch handles that + headers["content-encoding"] = "identity" + status_code = response_js.status + + # get a reader from the fetch response + body_stream_js = response_js.body.getReader() + return Response( + status_code=status_code, + headers=headers, + stream=EmscriptenStream(body_stream_js, read_timeout, abort_controller_js), + ) + + def _can_use_jspi(self) -> bool: + """Returns true if the pyodide environment allows for use + of synchronous javascript promise calls. If not we have to + fall back to the browser XMLHttpRequest api. + """ + # Ignore this next if statement from coverage because only one part + # will be run depending on the pyodide version + if hasattr(pyodide.ffi, "can_run_sync"): + return bool(pyodide.ffi.can_run_sync()) # pragma: no cover + else: + from pyodide_js._module import ( + validSuspender, + ) # pragma: no cover + + return bool(validSuspender.value) # pragma: no cover + + def _is_in_browser_main_thread(self) -> bool: + return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window + + def _no_jspi_fallback(self, request: Request) -> Response: + assert isinstance(request.stream, SyncByteStream) + try: + js_xhr = js.XMLHttpRequest.new() + + req_body: bytes | None = b"".join(request.stream) + if req_body is not None and len(req_body) == 0: + req_body = None + + timeout = 0.0 + if "timeout" in request.extensions: + timeout_dict = request.extensions["timeout"] + if timeout_dict is not None: + if "connect" in timeout_dict: + timeout = timeout_dict["connect"] + if "read" in timeout_dict: + timeout = timeout_dict["connect"] + + # XHMLHttpRequest only supports timeouts and proper + # binary file reading in web-workers + if not self._is_in_browser_main_thread(): + js_xhr.responseType = "arraybuffer" + if timeout > 0.0: + js_xhr.timeout = int(timeout * 1000) + else: + # this is a nasty hack to be able to read binary files on + # main browser thread using xmlhttprequest + js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15") + + js_xhr.open(request.method, request.url, False) + + for name, value in request.headers.items(): + if name.lower() not in HEADERS_TO_IGNORE: + js_xhr.setRequestHeader(name, value) + + js_xhr.send(pyodide.ffi.to_js(req_body)) + + headers = dict( + email.parser.Parser().parsestr(js_xhr.getAllResponseHeaders()) + ) + + if not self._is_in_browser_main_thread(): + body = js_xhr.response.to_py().tobytes() + else: + body = js_xhr.response.encode("ISO-8859-15") + + return Response(status_code=js_xhr.status, headers=headers, content=body) + except pyodide.ffi.JsException as err: + if err.name == "TimeoutError": + raise ConnectTimeout(message="Request timed out") + else: + raise ConnectError(message=err.message) + + def close(self) -> None: + pass # pragma: nocover + + +class AsyncEmscriptenStream(AsyncByteStream): + def __init__( + self, + response_stream_js: pyodide.ffi.JsProxy, + timeout: float, + abort_controller_js: pyodide.ffi.JsProxy, + ) -> None: + self._stream_js = response_stream_js + self.timeout = timeout + self.abort_controller_js = abort_controller_js + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + while self._stream_js is not None: + result_js = await _run_async_with_timeout( + self._stream_js.read(), + self.timeout, + self.abort_controller_js, + ReadTimeout, + ReadError, + ) + if result_js.done: + return + else: + this_buffer = result_js.value.to_py() + yield this_buffer + + async def aclose(self) -> None: + self._stream_js = None + + +class AsyncJavascriptFetchTransport(AsyncBaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + pass + + async def __aenter__(self: A) -> A: # Use generics for subclass support. + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + pass + + async def handle_async_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, AsyncByteStream) + body_data: bytes = b"" + async for x in request.stream: + body_data += x + if len(body_data) == 0: + req_body = None + else: + req_body = body_data + conn_timeout = 0.0 + read_timeout = 0.0 + if "timeout" in request.extensions: + timeout_dict = request.extensions["timeout"] + if timeout_dict is not None: + if "connect" in timeout_dict: + conn_timeout = timeout_dict["connect"] + if "read" in timeout_dict: + read_timeout = timeout_dict["connect"] + + abort_controller_js = js.AbortController.new() + headers = { + k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE + } + fetch_data = { + "headers": headers, + "body": pyodide.ffi.to_js(req_body), + "method": request.method, + "signal": abort_controller_js.signal, + } + + fetcher_promise_js = js.fetch( + request.url, + pyodide.ffi.to_js(fetch_data, dict_converter=js.Object.fromEntries), + ) + response_js = await _run_async_with_timeout( + fetcher_promise_js, + conn_timeout, + abort_controller_js, + ConnectTimeout, + ConnectError, + ) + + headers = {} + header_iter = response_js.headers.entries() + while True: + iter_value_js = header_iter.next() + if getattr(iter_value_js, "done", False): + break + else: + headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1]) + status_code = response_js.status + + # get a reader from the fetch response + body_stream_js = response_js.body.getReader() + return Response( + status_code=status_code, + headers=headers, + stream=AsyncEmscriptenStream( + body_stream_js, read_timeout, abort_controller_js + ), + ) + + async def aclose(self) -> None: + pass # pragma: nocover diff --git a/httpx/_urlparse.py b/httpx/_urlparse.py index bf190fd560..3166e86e12 100644 --- a/httpx/_urlparse.py +++ b/httpx/_urlparse.py @@ -22,8 +22,6 @@ import re import typing -import idna - from ._exceptions import InvalidURL MAX_URL_LENGTH = 65536 @@ -384,6 +382,7 @@ def encode_host(host: str) -> str: # reg-name = *( unreserved / pct-encoded / sub-delims ) WHATWG_SAFE = '"`{}%|\\' return quote(host.lower(), safe=SUB_DELIMS + WHATWG_SAFE) + import idna # IDNA hostnames try: diff --git a/httpx/_urls.py b/httpx/_urls.py index 147a8fa333..8c88b6b1a2 100644 --- a/httpx/_urls.py +++ b/httpx/_urls.py @@ -3,8 +3,6 @@ import typing from urllib.parse import parse_qs, unquote, urlencode -import idna - from ._types import QueryParamTypes from ._urlparse import urlparse from ._utils import primitive_value_to_str @@ -188,6 +186,8 @@ def host(self) -> str: host: str = self._uri_reference.host if host.startswith("xn--"): + import idna + host = idna.decode(host) return host diff --git a/mkdocs.yml b/mkdocs.yml index 86ca1e53b7..513e6a9e35 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,6 +36,7 @@ nav: - Transports: 'advanced/transports.md' - Text Encodings: 'advanced/text-encodings.md' - Extensions: 'advanced/extensions.md' + - Emscripten: 'advanced/emscripten.md' - Guides: - Async Support: 'async.md' - HTTP/2 Support: 'http2.md' diff --git a/pyproject.toml b/pyproject.toml index 675d2ad4c6..4cf9d4fac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,9 @@ classifiers = [ "Topic :: Internet :: WWW/HTTP", ] dependencies = [ - "certifi", - "httpcore==1.*", - "anyio", + "certifi;sys_platform != 'emscripten'", + "httpcore==1.*;sys_platform != 'emscripten'", + "anyio;sys_platform != 'emscripten'", "idna", ] dynamic = ["readme", "version"] @@ -131,3 +131,9 @@ markers = [ [tool.coverage.run] omit = ["venv/*"] include = ["httpx/*", "tests/*"] + +[tool.coverage.paths] +source = [ + "httpx/", + "**/httpx/", + ] \ No newline at end of file diff --git a/requirements-emscripten.txt b/requirements-emscripten.txt new file mode 100644 index 0000000000..1f2c646516 --- /dev/null +++ b/requirements-emscripten.txt @@ -0,0 +1,2 @@ +# extra requirements for testing on emscripten +pytest-pyodide==0.58.3 \ No newline at end of file diff --git a/scripts/coverage b/scripts/coverage index 25a2691074..f9d7293889 100755 --- a/scripts/coverage +++ b/scripts/coverage @@ -8,4 +8,13 @@ export SOURCE_FILES="httpx tests" set -x -${PREFIX}coverage report --show-missing --skip-covered --fail-under=100 +if [ -d 'pyodide_dist' ]; then + IGNORE_ARGS="--omit=venv/*" +else + # if we don't have a pyodide environment set up, then don't test coverage + # for the emscripten transport + IGNORE_ARGS="--omit=httpx/_transports/emscripten.py,tests/emscripten/*,venv/*" +fi + +${PREFIX}coverage combine +${PREFIX}coverage report ${IGNORE_ARGS} --show-missing --skip-covered --fail-under=100 diff --git a/scripts/docs b/scripts/docs index 4ac3beb7a6..d39206c445 100755 --- a/scripts/docs +++ b/scripts/docs @@ -1,4 +1,4 @@ -#!/bin/sh -e +#!/bin/bash -e export PREFIX="" if [ -d 'venv' ] ; then @@ -7,4 +7,17 @@ fi set -x +# for local use of emscripten, we need a test wheel +# in the docs folder so that the emscripten live demo +# web page works with the built version +function cleanup { + echo "Removing docs/test.whl" + rm docs/test.whl +} + +# remove the test wheel from docs after we serve, +# so it doesn't get added into the distribution on +# github docs +trap cleanup EXIT +cp dist/*.whl docs/test.whl ${PREFIX}mkdocs serve diff --git a/scripts/install b/scripts/install index 58b632403b..1e3b35f111 100755 --- a/scripts/install +++ b/scripts/install @@ -4,6 +4,7 @@ [ "$1" = "-p" ] && PYTHON=$2 || PYTHON="python3" REQUIREMENTS="requirements.txt" +REQUIREMENTS_EMSCRIPTEN="requirements-emscripten.txt" VENV="venv" set -x @@ -17,3 +18,7 @@ fi "$PIP" install -U pip "$PIP" install -r "$REQUIREMENTS" + +if [ -d 'pyodide_dist' ]; then + "$PIP" install -r "$REQUIREMENTS_EMSCRIPTEN" +fi diff --git a/scripts/test b/scripts/test index d778a544d2..6982f4e69c 100755 --- a/scripts/test +++ b/scripts/test @@ -1,8 +1,9 @@ #!/bin/sh -export PREFIX="" +# don't call this PREFIX, because that is used by node.js +export VENV_PREFIX="" if [ -d 'venv' ] ; then - export PREFIX="venv/bin/" + export VENV_PREFIX="venv/bin/" fi set -ex @@ -11,8 +12,16 @@ if [ -z $GITHUB_ACTIONS ]; then scripts/check fi -${PREFIX}coverage run -m pytest "$@" +# run all host tests first +${VENV_PREFIX}coverage run -p -m pytest "$@" --ignore=tests/emscripten + +if [ -d 'pyodide_dist' ]; then + # run emscripten specific tests on chrome and node.js (20+) + ${VENV_PREFIX}coverage run -p -m pytest -v --dist-dir=${PWD}/pyodide_dist --rt=chrome-no-host tests/emscripten/test_emscripten.py "$@" + ${VENV_PREFIX}coverage run -p -m pytest -v --dist-dir=${PWD}/pyodide_dist --rt=node-no-host tests/emscripten/test_emscripten.py "$@" +fi + if [ -z $GITHUB_ACTIONS ]; then - scripts/coverage + scripts/coverage fi diff --git a/tests/concurrency.py b/tests/concurrency.py index a8ed55829a..992aa99bfe 100644 --- a/tests/concurrency.py +++ b/tests/concurrency.py @@ -4,12 +4,12 @@ import asyncio -import sniffio import trio +from sniffio import current_async_library async def sleep(seconds: float) -> None: - if sniffio.current_async_library() == "trio": + if current_async_library() == "trio": await trio.sleep(seconds) # pragma: no cover else: await asyncio.sleep(seconds) diff --git a/tests/conftest.py b/tests/conftest.py index 858bca1397..1e229fe9a8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import threading import time import typing +from pathlib import Path import pytest import trustme @@ -72,6 +73,10 @@ async def app(scope: Scope, receive: Receive, send: Send) -> None: await redirect_301(scope, receive, send) elif scope["path"].startswith("/json"): await hello_world_json(scope, receive, send) + elif scope["path"].startswith("/wheel_download"): # pragma: nocover for emscripten + await wheel_download(scope, receive, send) + elif scope["path"].startswith("/emscripten"): # pragma: nocover for emscripten + await hello_world_emscripten(scope, receive, send) else: await hello_world(scope, receive, send) @@ -87,6 +92,55 @@ async def hello_world(scope: Scope, receive: Receive, send: Send) -> None: await send({"type": "http.response.body", "body": b"Hello, world!"}) +# For testing on emscripten, we require cross origin isolation headers +# to be set or else browsers won't be able to read from us from javascript +async def hello_world_emscripten( + scope: Scope, receive: Receive, send: Send +) -> None: # pragma: nocover for emscripten + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [ + [b"content-type", b"text/plain"], + [b"access-control-allow-origin", "*"], + [ + b"access-control-allow-methods", + b"PUT, GET, HEAD, POST, DELETE, OPTIONS", + ], + [b"Access-Control-Allow-Headers", b"*"], + ], + } + ) + await send({"type": "http.response.body", "body": b"Hello, world!"}) + + +# For testing on emscripten, it is useful to be able to +# get the wheel package so that we can install it e.g. +# on web-workers +async def wheel_download( + scope: Scope, receive: Receive, send: Send +) -> None: # pragma: nocover for emscripten + wheel_file = list(Path("dist").glob("*.whl"))[0] + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [ + [b"content-type", b"application/x-wheel"], + [b"access-control-allow-origin", "*"], + [ + b"access-control-allow-methods", + b"PUT, GET, HEAD, POST, DELETE, OPTIONS", + ], + [b"Access-Control-Allow-Headers", b"*"], + ], + } + ) + wheel_bytes = wheel_file.read_bytes() + await send({"type": "http.response.body", "body": wheel_bytes}) + + async def hello_world_json(scope: Scope, receive: Receive, send: Send) -> None: await send( { @@ -103,7 +157,16 @@ async def slow_response(scope: Scope, receive: Receive, send: Send) -> None: { "type": "http.response.start", "status": 200, - "headers": [[b"content-type", b"text/plain"]], + "headers": [ + [b"content-type", b"text/plain"], + [b"access-control-allow-origin", "*"], + [ + b"access-control-allow-methods", + b"PUT, GET, HEAD, POST, DELETE, OPTIONS", + ], + [b"Access-Control-Allow-Headers", b"*"], + [b"Cache-control", "no-store,private,no-cache,must-revalidate"], + ], } ) await sleep(1.0) # Allow triggering a read timeout. @@ -285,3 +348,21 @@ def server() -> typing.Iterator[TestServer]: config = Config(app=app, lifespan="off", loop="asyncio") server = TestServer(config=config) yield from serve_in_thread(server) + + +# a local https server is need for emscripten tests +# to work +@pytest.fixture(scope="session") +def https_server( + cert_pem_file, cert_private_key_file +): # pragma: nocover (only used by emscripten) + config = Config( + app=app, + lifespan="off", + ssl_certfile=cert_pem_file, + ssl_keyfile=cert_private_key_file, + port=8001, + loop="asyncio", + ) + server = TestServer(config=config) + yield from serve_in_thread(server) diff --git a/tests/emscripten/conftest.py b/tests/emscripten/conftest.py new file mode 100644 index 0000000000..6c01485ce6 --- /dev/null +++ b/tests/emscripten/conftest.py @@ -0,0 +1,156 @@ +# emscripten specific test fixtures + + +import random +import textwrap +from typing import Any, Generator, Type, TypeVar + +import pytest + +import httpx + +_coverage_count = 0 + + +def _get_coverage_filename(prefix: str) -> str: + global _coverage_count + _coverage_count += 1 + rand_part = "".join([random.choice("1234567890") for x in range(20)]) + return prefix + rand_part + f".{_coverage_count}" + + +@pytest.fixture() +def selenium_with_jspi_if_possible( + request: pytest.FixtureRequest, runtime: str, has_jspi: bool +) -> Generator[Any, None, None]: + if has_jspi: # pragma: no cover + fixture_name = "selenium_jspi" + else: + fixture_name = "selenium" + selenium_obj = request.getfixturevalue(fixture_name) + yield selenium_obj + + +T = TypeVar("T", bound=object) + + +def wrapRunner(wrapped: T, has_jspi: bool) -> T: + BaseType: Type[T] = type(wrapped) + + # need to ignore type of BaseType because it is dynamic + class CoverageRunner(BaseType): # type:ignore + COVERAGE_INIT_CODE = textwrap.dedent( + """ + import pyodide_js as pjs + await pjs.loadPackage("coverage") + from importlib.metadata import distribution + source_file = str(distribution('httpx').locate_file('httpx')) + import coverage + _coverage= coverage.Coverage(source=[source_file]) + _coverage.start() + """ + ) + + COVERAGE_TEARDOWN_CODE = textwrap.dedent( + """ + _coverage.stop() + _coverage.save() + _coverage_datafile = open(".coverage","rb") + _coverage_outdata = _coverage_datafile.read() + # avoid polluting main namespace too much + import js as _coverage_js + # convert to js Array (as default conversion is TypedArray which does + # bad things in firefox) + _coverage_js.Array.from_(_coverage_outdata) # last line is return value + """ + ) + + def __init__(self, base_runner: T, has_jspi: bool): + self.has_jspi = has_jspi + # copy attributes of base_runner + for k, v in base_runner.__dict__.items(): + self.__dict__[k] = v + + def _wrap_code(self, code: str, wheel_url: httpx.URL) -> str: + wrapped_code = ( + self.COVERAGE_INIT_CODE + + "import httpx\n" + + textwrap.dedent(code) + + self.COVERAGE_TEARDOWN_CODE + ) + if wheel_url: + wrapped_code = ( + "import pyodide_js as pjs\n" + + f"await pjs.loadPackage('{wheel_url}')\n" + + wrapped_code + ) + return wrapped_code + + def run_webworker_with_httpx(self, code: str, wheel_url: httpx.URL) -> None: + if self.browser == "node": + pytest.skip("Don't test web-workers in node.js") + + wrapped_code = self._wrap_code(code, wheel_url) + coverage_out_binary = bytes(self.run_webworker(wrapped_code)) + with open( + f"{_get_coverage_filename('.coverage.emscripten.')}", "wb" + ) as outfile: + outfile.write(coverage_out_binary) + + def run_with_httpx(self: Any, code: str, wheel_url: httpx.URL) -> None: + if self.browser == "node": + # stop node.js checking our https certificates + self.run_js('process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;') + wrapped_code = self._wrap_code(code, wheel_url) + print(wrapped_code) + coverage_out_binary = bytes(self.run_async(wrapped_code)) + with open( + f"{_get_coverage_filename('.coverage.emscripten.')}", "wb" + ) as outfile: + outfile.write(coverage_out_binary) + + return CoverageRunner(wrapped, has_jspi) + + +@pytest.fixture() +def pyodide_coverage( + selenium_with_jspi_if_possible: Any, has_jspi: bool +) -> Generator[Any, None, None]: + runner = wrapRunner(selenium_with_jspi_if_possible, has_jspi) + yield runner + + +@pytest.fixture(scope="session", params=["https", "http"]) +def server_url(request, server, https_server): + if request.param == "https": + yield https_server.url.copy_with(path="/emscripten") + else: + yield server.url.copy_with(path="/emscripten") + + +@pytest.fixture() +def wheel_url(server_url): + yield server_url.copy_with(path="/wheel_download/httpx.whl") + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Generate Webassembly Javascript Promise Integration based tests + only for platforms that support it. + + Currently: + 1) node.js only supports use of JSPI because it doesn't support + synchronous XMLHttpRequest + + 2) firefox doesn't support JSPI + + 3) Chrome supports JSPI on or off. + """ + if "has_jspi" in metafunc.fixturenames: + if metafunc.config.getoption("--runtime").startswith("node"): + metafunc.parametrize("has_jspi", [True]) + elif metafunc.config.getoption("--runtime").startswith( + "firefox" + ): # pragma: no cover + metafunc.parametrize("has_jspi", [False]) + else: + metafunc.parametrize("has_jspi", [True, False]) diff --git a/tests/emscripten/test_emscripten.py b/tests/emscripten/test_emscripten.py new file mode 100644 index 0000000000..03ae12c488 --- /dev/null +++ b/tests/emscripten/test_emscripten.py @@ -0,0 +1,212 @@ +from typing import Any + +import pytest + +import httpx + +# only run these tests if pytest_pyodide is installed +# so we don't break non-emscripten pytest running +pytest_pyodide = pytest.importorskip("pytest_pyodide") + +# make our ssl certificates work in chrome +pyodide_config = pytest_pyodide.config.get_global_config() +pyodide_config.set_flags( + "chrome", ["ignore-certificate-errors"] + pyodide_config.get_flags("chrome") +) + + +def test_get( + server_url: httpx.URL, wheel_url: httpx.URL, pyodide_coverage: Any +) -> None: + pyodide_coverage.run_with_httpx( + f""" + import httpx + response = httpx.get('{server_url}') + assert response.status_code == 200 + assert response.reason_phrase == "OK" + assert response.text == "Hello, world!" + assert response.http_version == "HTTP/1.1" + """, + wheel_url, + ) + + +def test_post_http( + server_url: httpx.URL, wheel_url: httpx.URL, pyodide_coverage: Any +) -> None: + pyodide_coverage.run_with_httpx( + f""" + import httpx + response = httpx.post('{server_url}', content=b"Hello, world!") + assert response.status_code == 200 + assert response.reason_phrase == "OK" + """, + wheel_url, + ) + + +def test_async_get( + server_url: httpx.URL, wheel_url: httpx.URL, pyodide_coverage: Any +) -> None: + pyodide_coverage.run_with_httpx( + f""" + import httpx + url = '{server_url}' + async with httpx.AsyncClient() as client: + response = await client.get(url) + assert response.status_code == 200 + assert response.text == "Hello, world!" + assert response.http_version == "HTTP/1.1" + assert response.headers + assert repr(response) == "" + """, + wheel_url, + ) + + +def test_async_get_timeout( + server_url: httpx.URL, + wheel_url: httpx.URL, + pyodide_coverage: Any, + request: pytest.FixtureRequest, +) -> None: + # test timeout on https and http + timeout_url = server_url.copy_with( + path="/slow_response", query=request.node.callspec.id.encode("UTF-8") + ) + pyodide_coverage.run_with_httpx( + f""" + import httpx + import pytest + url = '{timeout_url}' + with pytest.raises(httpx.TimeoutException): + async with httpx.AsyncClient() as client: + response = await client.get(url,timeout=0.1) + print(response.text) + """, + wheel_url, + ) + + +def test_sync_get_timeout( + server_url: httpx.URL, + wheel_url: httpx.URL, + pyodide_coverage: Any, + has_jspi: bool, + request: pytest.FixtureRequest, +) -> None: + # test timeout on https and http + if not has_jspi: + # if we are using XMLHttpRequest in a main browser thread then + # this will never timeout, or at least it will use the default + # browser timeout which is VERY long! + pytest.skip() + timeout_url = server_url.copy_with( + path="/slow_response", query=request.node.callspec.id.encode("UTF-8") + ) + pyodide_coverage.run_with_httpx( + f""" + import httpx + import pytest + url = '{timeout_url}' + with pytest.raises(httpx.TimeoutException): + response = httpx.get(url,timeout=0.1) + print(response.text) + """, + wheel_url, + ) + + +def test_sync_get_timeout_worker( + server_url: httpx.URL, + wheel_url: httpx.URL, + pyodide_coverage: Any, + request: pytest.FixtureRequest, +) -> None: + # test timeout on https and http + # this should timeout in 0.1 seconds + # (and shouldn't hit cache because of query string) + timeout_url = server_url.copy_with( + path="/slow_response", query=request.node.callspec.id.encode("UTF-8") + ) + pyodide_coverage.run_webworker_with_httpx( + f""" + import httpx + import pytest + url = '{timeout_url}' + with pytest.raises(httpx.TimeoutException): + response = httpx.get(url,timeout=0.1) + print(response.text) + + """, + wheel_url, + ) + + +def test_get_worker( + server_url: httpx.URL, wheel_url: httpx.URL, pyodide_coverage: Any +) -> None: + pyodide_coverage.run_webworker_with_httpx( + f""" + import httpx + response = httpx.get('{server_url}') + assert response.status_code == 200 + assert response.reason_phrase == "OK" + assert response.text == "Hello, world!" + 1 + """, + wheel_url, + ) + + +def test_async_get_error( + server_url: httpx.URL, wheel_url: httpx.URL, pyodide_coverage: Any +) -> None: + # test timeout on https and http + # 255.255.255.255 should always return an error + error_url = str(server_url).split(":")[0] + "://255.255.255.255/" + pyodide_coverage.run_with_httpx( + f""" + import httpx + import pytest + url = '{error_url}' + with pytest.raises(httpx.ConnectError): + response = httpx.get(url) + """, + wheel_url, + ) + + +def test_sync_get_error( + server_url: httpx.URL, wheel_url: httpx.URL, pyodide_coverage: Any +) -> None: + # test timeout on https and http + # 255.255.255.255 should always return an error + error_url = str(server_url).split(":")[0] + "://255.255.255.255/" + pyodide_coverage.run_with_httpx( + f""" + import httpx + import pytest + url = '{error_url}' + with pytest.raises(httpx.ConnectError): + async with httpx.AsyncClient(timeout=1.0) as client: + response = await client.get(url) + print(response.text) + """, + wheel_url, + ) + + +def test_async_post_json( + server_url: httpx.URL, wheel_url: httpx.URL, pyodide_coverage: Any +) -> None: + pyodide_coverage.run_with_httpx( + f""" + import httpx + url = '{server_url}' + async with httpx.AsyncClient() as client: + response = await client.post(url, json={{"text": "Hello, world!"}}) + assert response.status_code == 200 + """, + wheel_url, + ) diff --git a/tests/test_api.py b/tests/test_api.py index 225f384ede..cce48c055e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,3 +1,5 @@ +import subprocess +import sys import typing import pytest @@ -89,14 +91,12 @@ def test_get_invalid_url(): # check that httpcore isn't imported until we do a request def test_httpcore_lazy_loading(server): - import sys - - # unload our module if it is already loaded - if "httpx" in sys.modules: - del sys.modules["httpx"] - del sys.modules["httpcore"] - import httpx - - assert "httpcore" not in sys.modules - _response = httpx.get(server.url) - assert "httpcore" in sys.modules + subprocess.check_call( + [ + sys.executable, + "-c", + "import httpx,sys;assert 'httpcore' not in sys.modules;" + + f"_response = httpx.get('{server.url}');" + + "assert 'httpcore' in sys.modules", + ] + ) diff --git a/tests/test_config.py b/tests/test_config.py index 22abd4c22c..2adfb166f0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,6 @@ import ssl +import subprocess +import sys import typing from pathlib import Path @@ -182,3 +184,15 @@ def test_proxy_with_auth_from_url(): def test_invalid_proxy_scheme(): with pytest.raises(ValueError): httpx.Proxy("invalid://example.com") + + +def test_certifi_lazy_loading(): + subprocess.check_call( + [ + sys.executable, + "-c", + "import httpx,sys;assert 'certifi' not in sys.modules;" + "_context = httpx.create_ssl_context();" + "assert 'certifi' in sys.modules", + ] + )