Merge pull request #12300 from sbidoul/packaging-upgrade

Upgrade vendored packaging lib

Author: pradyunsg

news/12063.removal.rst

@@ -0,0 +1,4 @@
+Remove support for legacy versions and dependency specifiers. Packages with non
+standard-compliant versions or dependency specifiers are now ignored by the resolver.
+Already installed packages with non standard-compliant versions or dependency specifiers
+must be uninstalled before upgrading them.

news/packaging.vendor.rst

@@ -0,0 +1 @@
+Upgrade packaging to 24.0

news/pyparsing.vendor.rst

@@ -1 +1 @@
-Upgrade pyparsing to 3.1.2
+Remove pyparsing

src/pip/_internal/commands/check.py

@@ -7,7 +7,6 @@
 from pip._internal.operations.check import (
     check_package_set,
     create_package_set_from_installed,
-    warn_legacy_versions_and_specifiers,
 )
 from pip._internal.utils.misc import write_output
 
@@ -22,7 +21,6 @@ class CheckCommand(Command):
 
     def run(self, options: Values, args: List[str]) -> int:
         package_set, parsing_probs = create_package_set_from_installed()
-        warn_legacy_versions_and_specifiers(package_set)
         missing, conflicting = check_package_set(package_set)
 
         for project_name in missing:

src/pip/_internal/commands/download.py

@@ -139,7 +139,6 @@ def run(self, options: Values, args: List[str]) -> int:
                 downloaded.append(req.name)
 
         preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
-        requirement_set.warn_legacy_versions_and_specifiers()
 
         if downloaded:
             write_output("Successfully downloaded %s", " ".join(downloaded))

src/pip/_internal/commands/index.py

@@ -1,8 +1,8 @@
 import logging
 from optparse import Values
-from typing import Any, Iterable, List, Optional, Union
+from typing import Any, Iterable, List, Optional
 
-from pip._vendor.packaging.version import LegacyVersion, Version
+from pip._vendor.packaging.version import Version
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.req_command import IndexGroupCommand
@@ -115,7 +115,7 @@ def get_available_package_versions(self, options: Values, args: List[Any]) -> No
                 ignore_requires_python=options.ignore_requires_python,
             )
 
-            versions: Iterable[Union[LegacyVersion, Version]] = (
+            versions: Iterable[Version] = (
                 candidate.version for candidate in finder.find_all_candidates(query)
             )
 

src/pip/_internal/commands/install.py

@@ -387,9 +387,6 @@ def run(self, options: Values, args: List[str]) -> int:
                         json.dump(report.to_dict(), f, indent=2, ensure_ascii=False)
 
             if options.dry_run:
-                # In non dry-run mode, the legacy versions and specifiers check
-                # will be done as part of conflict detection.
-                requirement_set.warn_legacy_versions_and_specifiers()
                 would_install_items = sorted(
                     (r.metadata["name"], r.metadata["version"])
                     for r in requirement_set.requirements_to_install

src/pip/_internal/commands/list.py

@@ -4,6 +4,7 @@
 from typing import TYPE_CHECKING, Generator, List, Optional, Sequence, Tuple, cast
 
 from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging.version import Version
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.req_command import IndexGroupCommand
@@ -18,7 +19,6 @@
 from pip._internal.utils.misc import tabulate, write_output
 
 if TYPE_CHECKING:
-    from pip._internal.metadata.base import DistributionVersion
 
     class _DistWithLatestInfo(BaseDistribution):
         """Give the distribution object a couple of extra fields.
@@ -27,7 +27,7 @@ class _DistWithLatestInfo(BaseDistribution):
         makes the rest of the code much cleaner.
         """
 
-        latest_version: DistributionVersion
+        latest_version: Version
         latest_filetype: str
 
     _ProcessedDists = Sequence[_DistWithLatestInfo]
@@ -333,7 +333,7 @@ def format_for_columns(
     for proj in pkgs:
         # if we're working on the 'outdated' list, separate out the
         # latest_version and type
-        row = [proj.raw_name, str(proj.version)]
+        row = [proj.raw_name, proj.raw_version]
 
         if running_outdated:
             row.append(str(proj.latest_version))

src/pip/_internal/commands/show.py

@@ -2,6 +2,7 @@
 from optparse import Values
 from typing import Generator, Iterable, Iterator, List, NamedTuple, Optional
 
+from pip._vendor.packaging.requirements import InvalidRequirement
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.cli.base_command import Command
@@ -100,12 +101,19 @@ def _get_requiring_packages(current_dist: BaseDistribution) -> Iterator[str]:
         except KeyError:
             continue
 
-        requires = sorted(
-            # Avoid duplicates in requirements (e.g. due to environment markers).
-            {req.name for req in dist.iter_dependencies()},
-            key=str.lower,
-        )
-        required_by = sorted(_get_requiring_packages(dist), key=str.lower)
+        try:
+            requires = sorted(
+                # Avoid duplicates in requirements (e.g. due to environment markers).
+                {req.name for req in dist.iter_dependencies()},
+                key=str.lower,
+            )
+        except InvalidRequirement:
+            requires = sorted(dist.iter_raw_dependencies(), key=str.lower)
+
+        try:
+            required_by = sorted(_get_requiring_packages(dist), key=str.lower)
+        except InvalidRequirement:
+            required_by = ["#N/A"]
 
         try:
             entry_points_text = dist.read_text("entry_points.txt")
@@ -139,7 +147,7 @@ def _get_requiring_packages(current_dist: BaseDistribution) -> Iterator[str]:
 
         yield _PackageInfo(
             name=dist.raw_name,
-            version=str(dist.version),
+            version=dist.raw_version,
             location=dist.location or "",
             editable_project_location=dist.editable_project_location,
             requires=requires,

src/pip/_internal/commands/wheel.py

@@ -154,7 +154,6 @@ def run(self, options: Values, args: List[str]) -> int:
                 reqs_to_build.append(req)
 
         preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
-        requirement_set.warn_legacy_versions_and_specifiers()
 
         # build wheels
         build_successes, build_failures = build(

src/pip/_internal/exceptions.py

@@ -354,6 +354,17 @@ def __str__(self) -> str:
         )
 
 
+class MetadataInvalid(InstallationError):
+    """Metadata is invalid."""
+
+    def __init__(self, ireq: "InstallRequirement", error: str) -> None:
+        self.ireq = ireq
+        self.error = error
+
+    def __str__(self) -> str:
+        return f"Requested {self.ireq} has invalid metadata: {self.error}"
+
+
 class InstallationSubprocessError(DiagnosticPipError, InstallationError):
     """A subprocess call failed."""
 

src/pip/_internal/index/package_finder.py

@@ -11,7 +11,7 @@
 from pip._vendor.packaging import specifiers
 from pip._vendor.packaging.tags import Tag
 from pip._vendor.packaging.utils import canonicalize_name
-from pip._vendor.packaging.version import _BaseVersion
+from pip._vendor.packaging.version import InvalidVersion, _BaseVersion
 from pip._vendor.packaging.version import parse as parse_version
 
 from pip._internal.exceptions import (
@@ -752,11 +752,14 @@ def get_install_candidate(
             self._log_skipped_link(link, result, detail)
             return None
 
-        return InstallationCandidate(
-            name=link_evaluator.project_name,
-            link=link,
-            version=detail,
-        )
+        try:
+            return InstallationCandidate(
+                name=link_evaluator.project_name,
+                link=link,
+                version=detail,
+            )
+        except InvalidVersion:
+            return None
 
     def evaluate_links(
         self, link_evaluator: LinkEvaluator, links: Iterable[Link]

src/pip/_internal/metadata/base.py

@@ -25,7 +25,7 @@
 from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
-from pip._vendor.packaging.version import LegacyVersion, Version
+from pip._vendor.packaging.version import Version
 
 from pip._internal.exceptions import NoneMetadataError
 from pip._internal.locations import site_packages, user_site
@@ -41,8 +41,6 @@
 
 from ._json import msg_to_json
 
-DistributionVersion = Union[LegacyVersion, Version]
-
 InfoPath = Union[str, pathlib.PurePath]
 
 logger = logging.getLogger(__name__)
@@ -140,10 +138,10 @@ def from_wheel(cls, wheel: "Wheel", name: str) -> "BaseDistribution":
         raise NotImplementedError()
 
     def __repr__(self) -> str:
-        return f"{self.raw_name} {self.version} ({self.location})"
+        return f"{self.raw_name} {self.raw_version} ({self.location})"
 
     def __str__(self) -> str:
-        return f"{self.raw_name} {self.version}"
+        return f"{self.raw_name} {self.raw_version}"
 
     @property
     def location(self) -> Optional[str]:
@@ -274,7 +272,11 @@ def canonical_name(self) -> NormalizedName:
         raise NotImplementedError()
 
     @property
-    def version(self) -> DistributionVersion:
+    def version(self) -> Version:
+        raise NotImplementedError()
+
+    @property
+    def raw_version(self) -> str:
         raise NotImplementedError()
 
     @property
@@ -443,24 +445,19 @@ def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requiremen
         """
         raise NotImplementedError()
 
-    def iter_provided_extras(self) -> Iterable[str]:
+    def iter_raw_dependencies(self) -> Iterable[str]:
+        """Raw Requires-Dist metadata."""
+        return self.metadata.get_all("Requires-Dist", [])
+
+    def iter_provided_extras(self) -> Iterable[NormalizedName]:
         """Extras provided by this distribution.
 
         For modern .dist-info distributions, this is the collection of
         "Provides-Extra:" entries in distribution metadata.
 
-        The return value of this function is not particularly useful other than
-        display purposes due to backward compatibility issues and the extra
-        names being poorly normalized prior to PEP 685. If you want to perform
-        logic operations on extras, use :func:`is_extra_provided` instead.
-        """
-        raise NotImplementedError()
-
-    def is_extra_provided(self, extra: str) -> bool:
-        """Check whether an extra is provided by this distribution.
-
-        This is needed mostly for compatibility issues with pkg_resources not
-        following the extra normalization rules defined in PEP 685.
+        The return value of this function is expected to be normalised names,
+        per PEP 685, with the returned value being handled appropriately by
+        `iter_dependencies`.
         """
         raise NotImplementedError()
 

src/pip/_internal/metadata/importlib/_dists.py

@@ -16,13 +16,13 @@
 
 from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
+from pip._vendor.packaging.version import Version
 from pip._vendor.packaging.version import parse as parse_version
 
 from pip._internal.exceptions import InvalidWheel, UnsupportedWheel
 from pip._internal.metadata.base import (
     BaseDistribution,
     BaseEntryPoint,
-    DistributionVersion,
     InfoPath,
     Wheel,
 )
@@ -171,9 +171,13 @@ def canonical_name(self) -> NormalizedName:
         return canonicalize_name(name)
 
     @property
-    def version(self) -> DistributionVersion:
+    def version(self) -> Version:
         return parse_version(self._dist.version)
 
+    @property
+    def raw_version(self) -> str:
+        return self._dist.version
+
     def is_file(self, path: InfoPath) -> bool:
         return self._dist.read_text(str(path)) is not None
 
@@ -204,19 +208,18 @@ def _metadata_impl(self) -> email.message.Message:
         # until upstream can improve the protocol. (python/cpython#94952)
         return cast(email.message.Message, self._dist.metadata)
 
-    def iter_provided_extras(self) -> Iterable[str]:
-        return self.metadata.get_all("Provides-Extra", [])
-
-    def is_extra_provided(self, extra: str) -> bool:
-        return any(
-            canonicalize_name(provided_extra) == canonicalize_name(extra)
-            for provided_extra in self.metadata.get_all("Provides-Extra", [])
-        )
+    def iter_provided_extras(self) -> Iterable[NormalizedName]:
+        return [
+            canonicalize_name(extra)
+            for extra in self.metadata.get_all("Provides-Extra", [])
+        ]
 
     def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
         contexts: Sequence[Dict[str, str]] = [{"extra": e} for e in extras]
         for req_string in self.metadata.get_all("Requires-Dist", []):
-            req = Requirement(req_string)
+            # strip() because email.message.Message.get_all() may return a leading \n
+            # in case a long header was wrapped.
+            req = Requirement(req_string.strip())
             if not req.marker:
                 yield req
             elif not extras and req.marker.evaluate({"extra": ""}):