diff --git a/news/13986.feature.rst b/news/13986.feature.rst new file mode 100644 index 0000000000..70686f847c --- /dev/null +++ b/news/13986.feature.rst @@ -0,0 +1 @@ +Improve package candidate collection performance by optimizing URL parsing. diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 200ec34c56..ccfa5bf1e9 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from typing import ( Any, + Final, NamedTuple, ) @@ -140,6 +141,11 @@ def _clean_file_url_path(part: str) -> str: # percent-encoded: / _reserved_chars_re = re.compile("(@|%2F)", re.IGNORECASE) +# Characters that survive a quote(unquote(part)) round-trip unchanged in a +# URL path: quote()'s always-safe alphabet plus '/' (the default of the safe +# argument). +_UNSAFE_URL_PATH_CHARS_RE: Final[re.Pattern[str]] = re.compile(r"[^A-Za-z0-9_./~\-]") + def _clean_url_path(path: str, is_local_path: bool) -> str: """ @@ -169,6 +175,17 @@ def _ensure_quoted_url(url: str) -> str: For example, if ' ' occurs in the URL, it will be replaced with "%20", and without double-quoting other characters. """ + # Fast path: skip quoting round-trip if the path component of a http(s):// link + # only contains characters that quote() would leave untouched AND has no %-escapes + # for unquote(). + # + # NOTE: we check everything after the scheme because calling urlsplit() to get just + # the path component is too costly here. + if url.startswith(("https://", "http://")): + url_no_scheme = url.removeprefix("https:").removeprefix("http:") + if _UNSAFE_URL_PATH_CHARS_RE.search(url_no_scheme) is None: + return url + # Split the URL into parts according to the general structure # `scheme://netloc/path?query#fragment`. result = urllib.parse.urlsplit(url) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index b84fbdb0b3..eaa4f0d017 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -424,6 +424,36 @@ def test_ensure_quoted_url(url: str, clean_url: str) -> None: assert _ensure_quoted_url(url) == clean_url +@pytest.mark.parametrize( + "url", + [ + pytest.param( + "https://files.pythonhosted.org/packages/12/34/somepackage-1.2.3-py3-none-any.whl", + id="typical-pypi-wheel", + ), + pytest.param( + "https://files.pythonhosted.org/packages/12/34/somepackage-1.2.3-py3-none-any.whl#sha256=abc", + id="pypi-wheel-with-fragment", + ), + pytest.param( + "http://localhost:8181/simple/foo/", + id="http-localhost-with-port", + ), + pytest.param( + "https://example.com/path/to/file.tar.gz?build=1", + id="https-with-query", + ), + ], +) +def test_ensure_quoted_url_idempotent_for_clean_urls(url: str) -> None: + """http(s):// URLs that are pure ASCII with no whitespace and no + %-escapes already pass through urlsplit() + urlunsplit() unchanged. + The function MUST return the input unchanged for them so callers can rely + on it as an identity (the implementation may take a fast path here). + """ + assert _ensure_quoted_url(url) == url + + def _test_parse_links_data_attribute( anchor_html: str, attr: str, expected: str | None ) -> Link: