From 5beef23fc84338511dfe4311199e92251a0482a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Feb 2021 01:28:49 +0000 Subject: [PATCH] Use resolver in more cases. --- ognibuild/debian/apt.py | 66 ++++- ognibuild/debian/fix_build.py | 445 +++++++--------------------------- ognibuild/requirements.py | 182 ++++++++++++++ ognibuild/resolver/apt.py | 310 ++++++++++++++++++++++- 4 files changed, 627 insertions(+), 376 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 4f16ef7..e8a6934 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -29,8 +29,6 @@ from debian.deb822 import Release from .. import DetailedFailure from ..session import Session, run_with_tee -from .build import get_build_architecture - class UnidentifiedError(Exception): @@ -64,6 +62,12 @@ class AptManager(object): def __init__(self, session): self.session = session + def package_exists(self, package: str) -> bool: + raise NotImplementedError(self.package_exists) + + def get_package_for_paths(self, paths, regex=False): + raise NotImplementedError(self.get_package_for_paths) + def missing(self, packages): root = getattr(self.session, "location", "/") status_path = os.path.join(root, "var/lib/dpkg/status") @@ -80,6 +84,7 @@ class AptManager(object): return list(missing) def install(self, packages: List[str]) -> None: + logging.info('Installing using apt: %r', packages) packages = self.missing(packages) if packages: run_apt(self.session, ["install"] + packages) @@ -88,8 +93,30 @@ class AptManager(object): run_apt(self.session, ["satisfy"] + deps) +class LocalAptManager(AptManager): + + def __init__(self): + from ..session.plain import PlainSession + self.session = PlainSession() + self._apt_cache = None + + def package_exists(self, package): + if self._apt_cache is None: + import apt_pkg + + self._apt_cache = apt_pkg.Cache() + for p in self._apt_cache.packages: + if p.name == package: + return True + return False + + def get_package_for_paths(self, paths, regex=False): + # TODO(jelmer): Make sure we use whatever is configured in self.session + return get_package_for_paths(paths, regex=regex) + + class FileSearcher(object): - def search_files(self, path, regex=False): + def search_files(self, path: str, regex: bool = False) -> Iterator[str]: raise NotImplementedError(self.search_files) @@ -98,9 +125,6 @@ class ContentsFileNotFound(Exception): class AptContentsFileSearcher(FileSearcher): - - _user_agent = 'ognibuild/0.1' - def __init__(self): self._db = {} @@ -137,6 +161,7 @@ class AptContentsFileSearcher(FileSearcher): @classmethod def from_repositories(cls, sources): + from .debian.build import get_build_architecture # TODO(jelmer): Verify signatures, etc. urls = [] arches = [get_build_architecture(), "all"] @@ -159,11 +184,11 @@ class AptContentsFileSearcher(FileSearcher): urls.append("%s/%s/%s" % (base_url, name, entry["name"])) return cls.from_urls(urls) - @classmethod - def _get(cls, url): + @staticmethod + def _get(url): from urllib.request import urlopen, Request - request = Request(url, headers={"User-Agent": cls._user_agent}) + request = Request(url, headers={"User-Agent": "Debian Janitor"}) return urlopen(request) def load_url(self, url): @@ -192,7 +217,7 @@ class GeneratedFileSearcher(FileSearcher): def __init__(self, db): self._db = db - def search_files(self, path, regex=False): + def search_files(self, path: str, regex: bool = False) -> Iterator[str]: for p, pkg in sorted(self._db.items()): if regex: if re.match(path, p): @@ -215,7 +240,7 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( _apt_file_searcher = None -def search_apt_file(path: str, regex: bool = False) -> Iterator[FileSearcher]: +def search_apt_file(path: str, regex: bool = False) -> Iterator[str]: global _apt_file_searcher if _apt_file_searcher is None: # TODO(jelmer): cache file @@ -223,3 +248,22 @@ def search_apt_file(path: str, regex: bool = False) -> Iterator[FileSearcher]: if _apt_file_searcher: yield from _apt_file_searcher.search_files(path, regex=regex) yield from GENERATED_FILE_SEARCHER.search_files(path, regex=regex) + + +def get_package_for_paths(paths: List[str], regex: bool = False) -> Optional[str]: + candidates: Set[str] = set() + for path in paths: + candidates.update(search_apt_file(path, regex=regex)) + if candidates: + break + if len(candidates) == 0: + logging.warning("No packages found that contain %r", paths) + return None + if len(candidates) > 1: + logging.warning( + "More than 1 packages found that contain %r: %r", path, candidates + ) + # Euhr. Pick the one with the shortest name? + return sorted(candidates, key=len)[0] + else: + return candidates.pop() diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 71ee188..d72721f 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -21,7 +21,6 @@ __all__ = [ import logging import os -import subprocess import sys from typing import List, Callable, Type, Tuple, Set, Optional @@ -106,9 +105,35 @@ from buildlog_consultant.sbuild import ( SbuildFailure, ) -from ..apt import AptManager, LocalAptManager -from ..resolver.apt import AptResolver -from ..requirements import BinaryRequirement +from .apt import AptManager, LocalAptManager +from ..resolver.apt import ( + AptResolver, + NoAptPackage, + get_package_for_python_module, + ) +from ..requirements import ( + BinaryRequirement, + PathRequirement, + PkgConfigRequirement, + CHeaderRequirement, + JavaScriptRuntimeRequirement, + ValaPackageRequirement, + RubyGemRequirement, + GoPackageRequirement, + DhAddonRequirement, + PhpClassRequirement, + RPackageRequirement, + NodePackageRequirement, + LibraryRequirement, + RubyFileRequirement, + XmlEntityRequirement, + SprocketsFileRequirement, + JavaClassRequirement, + HaskellPackageRequirement, + MavenArtifactRequirement, + GnomeCommonRequirement, + JDKFileRequirement, + ) from .build import attempt_build, DEFAULT_BUILDER @@ -148,7 +173,7 @@ class DependencyContext(object): class BuildDependencyContext(DependencyContext): - def add_dependency(self, package, minimum_version=None): + def add_dependency(self, package: str, minimum_version: Optional[Version] = None): return add_build_dependency( self.tree, package, @@ -181,12 +206,12 @@ class AutopkgtestDependencyContext(DependencyContext): def add_build_dependency( - tree, - package, - minimum_version=None, - committer=None, - subpath="", - update_changelog=True, + tree: Tree, + package: str, + minimum_version: Optional[Version] = None, + committer: Optional[str] = None, + subpath: str = "", + update_changelog: bool = True, ): if not isinstance(package, str): raise TypeError(package) @@ -305,62 +330,6 @@ def commit_debian_changes( return True -def get_package_for_python_module(apt, module, python_version): - if python_version == "python3": - paths = [ - os.path.join( - "/usr/lib/python3/dist-packages", - module.replace(".", "/"), - "__init__.py", - ), - os.path.join( - "/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py" - ), - os.path.join( - "/usr/lib/python3\\.[0-9]+/lib-dynload", - module.replace(".", "/") + "\\.cpython-.*\\.so", - ), - os.path.join( - "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py" - ), - os.path.join( - "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py" - ), - ] - elif python_version == "python2": - paths = [ - os.path.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - module.replace(".", "/"), - "__init__.py", - ), - os.path.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - module.replace(".", "/") + ".py", - ), - os.path.join( - "/usr/lib/python2.\\.[0-9]/lib-dynload", - module.replace(".", "/") + ".so", - ), - ] - elif python_version == "pypy": - paths = [ - os.path.join( - "/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py" - ), - os.path.join( - "/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py" - ), - os.path.join( - "/usr/lib/pypy/dist-packages", - module.replace(".", "/") + "\\.pypy-.*\\.so", - ), - ] - else: - raise AssertionError("unknown python version %r" % python_version) - return apt.get_package_for_paths(paths, regex=True) - - def targeted_python_versions(tree: Tree) -> Set[str]: with tree.get_file("debian/control") as f: control = Deb822(f) @@ -378,13 +347,6 @@ def targeted_python_versions(tree: Tree) -> Set[str]: return targeted -def fix_missing_javascript_runtime(error, context): - package = context.apt.get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False) - if package is None: - return False - return context.add_dependency(package) - - def fix_missing_python_distribution(error, context): # noqa: C901 targeted = targeted_python_versions(context.tree) default = not targeted @@ -496,62 +458,60 @@ def fix_missing_python_module(error, context): return True -def fix_missing_go_package(error, context): - package = context.apt.get_package_for_paths( - [os.path.join("/usr/share/gocode/src", error.package, ".*")], regex=True - ) - if package is None: - return False - return context.add_dependency(package) - - -def fix_missing_c_header(error, context): - package = context.apt.get_package_for_paths( - [os.path.join("/usr/include", error.header)], regex=False - ) - if package is None: - package = context.apt.get_package_for_paths( - [os.path.join("/usr/include", ".*", error.header)], regex=True - ) - if package is None: - return False - return context.add_dependency(package) - - -def fix_missing_pkg_config(error, context): - package = context.apt.get_package_for_paths( - [os.path.join("/usr/lib/pkgconfig", error.module + ".pc")] - ) - if package is None: - package = context.apt.get_package_for_paths( - [os.path.join("/usr/lib", ".*", "pkgconfig", error.module + ".pc")], - regex=True, - ) - if package is None: - return False - return context.add_dependency(package, minimum_version=error.minimum_version) - - -def fix_missing_command(error, context): - package = context.resolve_apt(BinaryRequirement(error.command)) - return context.add_dependency(package) - - -def fix_missing_file(error, context): - package = context.apt.get_package_for_paths([error.path]) - if package is None: - return False - return context.add_dependency(package) - - -def fix_missing_sprockets_file(error, context): - if error.content_type == "application/javascript": - path = "/usr/share/.*/app/assets/javascripts/%s.js$" % error.name +def fix_missing_requirement(error, context): + if isinstance(error, MissingFile): + req = PathRequirement(error.path) + elif isinstance(error, MissingCommand): + req = BinaryRequirement(error.command) + elif isinstance(error, MissingPkgConfig): + req = PkgConfigRequirement( + error.module, error.minimum_version) + elif isinstance(error, MissingCHeader): + req = CHeaderRequirement(error.header) + elif isinstance(error, MissingJavaScriptRuntime): + req = JavaScriptRuntimeRequirement() + elif isinstance(error, MissingRubyGem): + req = RubyGemRequirement(error.gem, error.version) + elif isinstance(error, MissingValaPackage): + req = ValaPackageRequirement(error.package) + elif isinstance(error, MissingGoPackage): + req = GoPackageRequirement(error.package) + elif isinstance(error, DhAddonLoadFailure): + req = DhAddonRequirement(error.path) + elif isinstance(error, MissingPhpClass): + req = PhpClassRequirement(error.php_class) + elif isinstance(error, MissingRPackage): + req = RPackageRequirement(error.package, error.minimum_version) + elif isinstance(error, MissingNodeModule): + req = NodePackageRequirement(error.module) + elif isinstance(error, MissingLibrary): + req = LibraryRequirement(error.library) + elif isinstance(error, MissingRubyFile): + req = RubyFileRequirement(error.filename) + elif isinstance(error, MissingXmlEntity): + req = XmlEntityRequirement(error.url) + elif isinstance(error, MissingSprocketsFile): + req = SprocketsFileRequirement(error.content_type, error.name) + elif isinstance(error, MissingJavaClass): + req = JavaClassRequirement(error.classname) + elif isinstance(error, MissingHaskellDependencies): + # TODO(jelmer): Create multiple HaskellPackageRequirement objects? + req = HaskellPackageRequirement(error.package) + elif isinstance(error, MissingMavenArtifacts): + # TODO(jelmer): Create multiple MavenArtifactRequirement objects? + req = MavenArtifactRequirement(error.artifacts) + elif isinstance(error, MissingCSharpCompiler): + req = BinaryRequirement('msc') + elif isinstance(error, GnomeCommonMissing): + req = GnomeCommonRequirement() + elif isinstance(error, MissingJDKFile): + req = JDKFileRequirement(error.jdk_path, error.filename) else: - logging.warning("unable to handle content type %s", error.content_type) - return False - package = context.apt.get_package_for_paths([path], regex=True) - if package is None: + return None + + try: + package = context.resolve_apt(req) + except NoAptPackage: return False return context.add_dependency(package) @@ -595,159 +555,10 @@ def fix_missing_perl_file(error, context): return context.add_dependency(package) -def get_package_for_node_package(apt, node_package): - paths = [ - "/usr/share/nodejs/.*/node_modules/%s/package.json" % node_package, - "/usr/lib/nodejs/%s/package.json" % node_package, - "/usr/share/nodejs/%s/package.json" % node_package, - ] - return apt.get_package_for_paths(paths, regex=True) - - -def fix_missing_node_module(error, context): - package = get_package_for_node_package(context.apt, error.module) - if package is None: - logging.warning("no node package found for %s.", error.module) - return False - return context.add_dependency(package) - - -def fix_missing_dh_addon(error, context): - paths = [os.path.join("/usr/share/perl5", error.path)] - package = context.apt.get_package_for_paths(paths) - if package is None: - logging.warning("no package for debhelper addon %s", error.name) - return False - return context.add_dependency(package) - - def retry_apt_failure(error, context): return True -def fix_missing_php_class(error, context): - path = "/usr/share/php/%s.php" % error.php_class.replace("\\", "/") - package = context.apt.get_package_for_paths([path]) - if package is None: - logging.warning("no package for PHP class %s", error.php_class) - return False - return context.add_dependency(package) - - -def fix_missing_jdk_file(error, context): - path = error.jdk_path + ".*/" + error.filename - package = context.apt.get_package_for_paths([path], regex=True) - if package is None: - logging.warning( - "no package found for %s (JDK: %s) - regex %s", - error.filename, - error.jdk_path, - path, - ) - return False - return context.add_dependency(package) - - -def fix_missing_vala_package(error, context): - path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % error.package - package = context.apt.get_package_for_paths([path], regex=True) - if package is None: - logging.warning("no file found for package %s - regex %s", error.package, path) - return False - return context.add_dependency(package) - - -def fix_missing_xml_entity(error, context): - # Ideally we should be using the XML catalog for this, but hardcoding - # a few URLs will do for now.. - URL_MAP = { - "http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/" - } - for url, path in URL_MAP.items(): - if error.url.startswith(url): - search_path = os.path.join(path, error.url[len(url) :]) - break - else: - return False - - package = context.apt.get_package_for_paths([search_path], regex=False) - if package is None: - return False - return context.add_dependency(package) - - -def fix_missing_library(error, context): - paths = [ - os.path.join("/usr/lib/lib%s.so$" % error.library), - os.path.join("/usr/lib/.*/lib%s.so$" % error.library), - os.path.join("/usr/lib/lib%s.a$" % error.library), - os.path.join("/usr/lib/.*/lib%s.a$" % error.library), - ] - package = context.apt.get_package_for_paths(paths, regex=True) - if package is None: - logging.warning("no package for library %s", error.library) - return False - return context.add_dependency(package) - - -def fix_missing_ruby_gem(error, context): - paths = [ - os.path.join( - "/usr/share/rubygems-integration/all/" - "specifications/%s-.*\\.gemspec" % error.gem - ) - ] - package = context.apt.get_package_for_paths(paths, regex=True) - if package is None: - logging.warning("no package for gem %s", error.gem) - return False - return context.add_dependency(package, minimum_version=error.version) - - -def fix_missing_ruby_file(error, context): - paths = [os.path.join("/usr/lib/ruby/vendor_ruby/%s.rb" % error.filename)] - package = context.apt.get_package_for_paths(paths) - if package is not None: - return context.add_dependency(package) - paths = [ - os.path.join( - r"/usr/share/rubygems-integration/all/gems/([^/]+)/" - "lib/%s.rb" % error.filename - ) - ] - package = context.apt.get_package_for_paths(paths, regex=True) - if package is not None: - return context.add_dependency(package) - - logging.warning("no package for ruby file %s", error.filename) - return False - - -def fix_missing_r_package(error, context): - paths = [os.path.join("/usr/lib/R/site-library/.*/R/%s$" % error.package)] - package = context.apt.get_package_for_paths(paths, regex=True) - if package is None: - logging.warning("no package for R package %s", error.package) - return False - return context.add_dependency(package, minimum_version=error.minimum_version) - - -def fix_missing_java_class(error, context): - # Unfortunately this only finds classes in jars installed on the host - # system :( - output = subprocess.check_output(["java-propose-classpath", "-c" + error.classname]) - classpath = [p for p in output.decode().strip(":").strip().split(":") if p] - if not classpath: - logging.warning("unable to find classpath for %s", error.classname) - return False - logging.info("Classpath for %s: %r", error.classname, classpath) - package = context.apt.get_package_for_paths(classpath) - if package is None: - logging.warning("no package for files in %r", classpath) - return False - return context.add_dependency(package) - - def enable_dh_autoreconf(context): # Debhelper >= 10 depends on dh-autoreconf and enables autoreconf by # default. @@ -768,9 +579,8 @@ def enable_dh_autoreconf(context): def fix_missing_configure(error, context): - if not context.tree.has_filename("configure.ac") and not context.tree.has_filename( - "configure.in" - ): + if (not context.tree.has_filename("configure.ac") and + not context.tree.has_filename("configure.in")): return False return enable_dh_autoreconf(context) @@ -783,43 +593,6 @@ def fix_missing_automake_input(error, context): return enable_dh_autoreconf(context) -def fix_missing_maven_artifacts(error, context): - artifact = error.artifacts[0] - parts = artifact.split(":") - if len(parts) == 4: - (group_id, artifact_id, kind, version) = parts - regex = False - elif len(parts) == 3: - (group_id, artifact_id, version) = parts - kind = "jar" - regex = False - elif len(parts) == 2: - version = ".*" - (group_id, artifact_id) = parts - kind = "jar" - regex = True - else: - raise AssertionError("invalid number of parts to artifact %s" % artifact) - paths = [ - os.path.join( - "/usr/share/maven-repo", - group_id.replace(".", "/"), - artifact_id, - version, - "%s-%s.%s" % (artifact_id, version, kind), - ) - ] - package = context.apt.get_package_for_paths(paths, regex=regex) - if package is None: - logging.warning("no package for artifact %s", artifact) - return False - return context.add_dependency(package) - - -def install_gnome_common(error, context): - return context.add_dependency("gnome-common") - - def install_gnome_common_dep(error, context): if error.package == "glib-gettext": package = context.apt.get_package_for_paths(["/usr/bin/glib-gettextize"]) @@ -914,19 +687,6 @@ def fix_missing_autoconf_macro(error, context): return context.add_dependency(package) -def fix_missing_c_sharp_compiler(error, context): - return context.add_dependency("mono-mcs") - - -def fix_missing_haskell_dependencies(error, context): - path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % error.deps[0][0] - package = context.apt.get_package_for_paths([path], regex=True) - if package is None: - logging.warning("no package for macro file %s", path) - return False - return context.add_dependency(package) - - VERSIONED_PACKAGE_FIXERS: List[ Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]] ] = [ @@ -939,35 +699,14 @@ VERSIONED_PACKAGE_FIXERS: List[ APT_FIXERS: List[Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [ (MissingPythonModule, fix_missing_python_module), (MissingPythonDistribution, fix_missing_python_distribution), - (MissingCHeader, fix_missing_c_header), - (MissingPkgConfig, fix_missing_pkg_config), - (MissingCommand, fix_missing_command), - (MissingFile, fix_missing_file), - (MissingSprocketsFile, fix_missing_sprockets_file), - (MissingGoPackage, fix_missing_go_package), (MissingPerlFile, fix_missing_perl_file), (MissingPerlModule, fix_missing_perl_file), - (MissingXmlEntity, fix_missing_xml_entity), - (MissingNodeModule, fix_missing_node_module), - (MissingRubyGem, fix_missing_ruby_gem), - (MissingRPackage, fix_missing_r_package), - (MissingLibrary, fix_missing_library), - (MissingJavaClass, fix_missing_java_class), - (DhAddonLoadFailure, fix_missing_dh_addon), - (MissingPhpClass, fix_missing_php_class), (AptFetchFailure, retry_apt_failure), - (MissingMavenArtifacts, fix_missing_maven_artifacts), - (GnomeCommonMissing, install_gnome_common), (MissingGnomeCommonDependency, install_gnome_common_dep), (MissingXfceDependency, install_xfce_dep), (MissingConfigStatusInput, fix_missing_config_status_input), - (MissingJDKFile, fix_missing_jdk_file), - (MissingRubyFile, fix_missing_ruby_file), - (MissingJavaScriptRuntime, fix_missing_javascript_runtime), (MissingAutoconfMacro, fix_missing_autoconf_macro), - (MissingValaPackage, fix_missing_vala_package), - (MissingCSharpCompiler, fix_missing_c_sharp_compiler), - (MissingHaskellDependencies, fix_missing_haskell_dependencies), + (Problem, fix_missing_requirement), ] diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 65bf1d5..98d929c 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -16,6 +16,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import posixpath +from typing import Optional, List, Tuple + from . import UpstreamRequirement @@ -62,3 +65,182 @@ class CargoCrateRequirement(UpstreamRequirement): def __init__(self, crate): super(CargoCrateRequirement, self).__init__('cargo-crate') self.crate = crate + + +class PkgConfigRequirement(UpstreamRequirement): + + module: str + + def __init__(self, module, minimum_version=None): + super(PkgConfigRequirement, self).__init__('pkg-config') + self.module = module + self.minimum_version = minimum_version + + +class PathRequirement(UpstreamRequirement): + + path: str + + def __init__(self, path): + super(PathRequirement, self).__init__('path') + self.path = path + + +class CHeaderRequirement(UpstreamRequirement): + + header: str + + def __init__(self, header): + super(CHeaderRequirement, self).__init__('c-header') + self.header = header + + +class JavaScriptRuntimeRequirement(UpstreamRequirement): + + def __init__(self): + super(JavaScriptRuntimeRequirement, self).__init__( + 'javascript-runtime') + + +class ValaPackageRequirement(UpstreamRequirement): + + package: str + + def __init__(self, package: str): + super(ValaPackageRequirement, self).__init__('vala') + self.package = package + + +class RubyGemRequirement(UpstreamRequirement): + + gem: str + minimum_version: Optional[str] + + def __init__(self, gem: str, minimum_version: Optional[str]): + super(RubyGemRequirement, self).__init__('gem') + self.gem = gem + self.minimum_version = minimum_version + + +class GoPackageRequirement(UpstreamRequirement): + + package: str + + def __init__(self, package: str): + super(GoPackageRequirement, self).__init__('go') + self.package = package + + +class DhAddonRequirement(UpstreamRequirement): + + path: str + + def __init__(self, path: str): + super(DhAddonRequirement, self).__init__('dh-addon') + self.path = path + + +class PhpClassRequirement(UpstreamRequirement): + + php_class: str + + def __init__(self, php_class: str): + super(PhpClassRequirement, self).__init__('php-class') + self.php_class = php_class + + +class RPackageRequirement(UpstreamRequirement): + + package: str + minimum_version: Optional[str] + + def __init__(self, package: str, minimum_version: Optional[str] = None): + super(RPackageRequirement, self).__init__('r-package') + self.package = package + self.minimum_version = minimum_version + + +class LibraryRequirement(UpstreamRequirement): + + library: str + + def __init__(self, library: str): + super(LibraryRequirement, self).__init__('lib') + self.library = library + + +class RubyFileRequirement(UpstreamRequirement): + + filename: str + + def __init__(self, filename: str): + super(RubyFileRequirement, self).__init__('ruby-file') + self.filename = filename + + +class XmlEntityRequirement(UpstreamRequirement): + + url: str + + def __init__(self, url: str): + super(XmlEntityRequirement, self).__init__('xml-entity') + self.url = url + + +class SprocketsFileRequirement(UpstreamRequirement): + + content_type: str + name: str + + def __init__(self, content_type: str, name: str): + super(SprocketsFileRequirement, self).__init__('sprockets-file') + self.content_type = content_type + self.name = name + + +class JavaClassRequirement(UpstreamRequirement): + + classname: str + + def __init__(self, classname: str): + super(JavaClassRequirement, self).__init__('java-class') + self.classname = classname + + +class HaskellPackageRequirement(UpstreamRequirement): + + package: str + + def __init__(self, package: str): + super(HaskellPackageRequirement, self).__init__('haskell-package') + self.package = package + + +class MavenArtifactRequirement(UpstreamRequirement): + + artifacts: List[Tuple[str, str, str]] + + def __init__(self, artifacts): + super(MavenArtifactRequirement, self).__init__('maven-artifact') + self.artifacts = artifacts + + +class GnomeCommonRequirement(UpstreamRequirement): + + def __init__(self): + super(GnomeCommonRequirement, self).__init__('gnome-common') + + +class JDKFileRequirement(UpstreamRequirement): + + jdk_path: str + filename: str + + def __init__(self, jdk_path: str, filename: str): + super(JDKFileRequirement, self).__init__('jdk-file') + self.jdk_path = jdk_path + self.filename = filename + + @property + def path(self): + return posixpath.join(self.jdk_path, self.filename) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index df1adb2..7d8e444 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -15,13 +15,35 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import logging import posixpath -from ..apt import AptManager +from ..debian.apt import AptManager from . import Resolver from ..requirements import ( BinaryRequirement, + CHeaderRequirement, + PkgConfigRequirement, + PathRequirement, + UpstreamRequirement, + JavaScriptRuntimeRequirement, + ValaPackageRequirement, + RubyGemRequirement, + GoPackageRequirement, + DhAddonRequirement, + PhpClassRequirement, + RPackageRequirement, + NodePackageRequirement, + LibraryRequirement, + RubyFileRequirement, + XmlEntityRequirement, + SprocketsFileRequirement, + JavaClassRequirement, + HaskellPackageRequirement, + MavenArtifactRequirement, + GnomeCommonRequirement, + JDKFileRequirement, ) @@ -29,6 +51,62 @@ class NoAptPackage(Exception): """No apt package.""" +def get_package_for_python_module(apt_mgr, module, python_version): + if python_version == "python3": + paths = [ + posixpath.join( + "/usr/lib/python3/dist-packages", + module.replace(".", "/"), + "__init__.py", + ), + posixpath.join( + "/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py" + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/lib-dynload", + module.replace(".", "/") + "\\.cpython-.*\\.so", + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py" + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py" + ), + ] + elif python_version == "python2": + paths = [ + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist-packages", + module.replace(".", "/"), + "__init__.py", + ), + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist-packages", + module.replace(".", "/") + ".py", + ), + posixpath.join( + "/usr/lib/python2.\\.[0-9]/lib-dynload", + module.replace(".", "/") + ".so", + ), + ] + elif python_version == "pypy": + paths = [ + posixpath.join( + "/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py" + ), + posixpath.join( + "/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py" + ), + posixpath.join( + "/usr/lib/pypy/dist-packages", + module.replace(".", "/") + "\\.pypy-.*\\.so", + ), + ] + else: + raise AssertionError("unknown python version %r" % python_version) + return apt_mgr.get_package_for_paths(paths, regex=True) + + def resolve_binary_req(apt_mgr, req): if posixpath.isabs(req.binary_name): paths = [req.binary_name] @@ -40,8 +118,218 @@ def resolve_binary_req(apt_mgr, req): return apt_mgr.get_package_for_paths(paths) +def resolve_pkg_config_req(apt_mgr, req): + package = apt_mgr.get_package_for_paths( + [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")], + req.minimum_version + ) + if package is None: + package = apt_mgr.get_package_for_paths( + [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], + regex=True, + minimum_version=req.minimum_version) + return package + + +def resolve_path_req(apt_mgr, req): + return apt_mgr.get_package_for_paths([req.path]) + + +def resolve_c_header_req(apt_mgr, req): + package = apt_mgr.get_package_for_paths( + [posixpath.join("/usr/include", req.header)], regex=False + ) + if package is None: + package = apt_mgr.get_package_for_paths( + [posixpath.join("/usr/include", ".*", req.header)], regex=True + ) + return package + + +def resolve_js_runtime_req(apt_mgr, req): + return apt_mgr.get_package_for_paths( + ["/usr/bin/node", "/usr/bin/duk"], regex=False) + + +def resolve_vala_package_req(apt_mgr, req): + path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package + return apt_mgr.get_package_for_paths([path], regex=True) + + +def resolve_ruby_gem_req(apt_mgr, req): + paths = [ + posixpath.join( + "/usr/share/rubygems-integration/all/" + "specifications/%s-.*\\.gemspec" % req.gem + ) + ] + return apt_mgr.get_package_for_paths( + paths, regex=True, minimum_version=req.minimum_version) + + +def resolve_go_package_req(apt_mgr, req): + return apt_mgr.get_package_for_paths( + [posixpath.join("/usr/share/gocode/src", req.package, ".*")], + regex=True + ) + + +def resolve_dh_addon_req(apt_mgr, req): + paths = [posixpath.join("/usr/share/perl5", req.path)] + return apt_mgr.get_package_for_paths(paths) + + +def resolve_php_class_req(apt_mgr, req): + path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") + return apt_mgr.get_package_for_paths([path]) + + +def resolve_r_package_req(apt_mgr, req): + paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)] + return apt_mgr.get_package_for_paths(paths, regex=True) + + +def resolve_node_package_req(apt_mgr, req): + paths = [ + "/usr/share/nodejs/.*/node_modules/%s/package.json" % req.package, + "/usr/lib/nodejs/%s/package.json" % req.package, + "/usr/share/nodejs/%s/package.json" % req.package, + ] + return apt_mgr.get_package_for_paths(paths, regex=True) + + +def resolve_library_req(apt_mgr, req): + paths = [ + posixpath.join("/usr/lib/lib%s.so$" % req.library), + posixpath.join("/usr/lib/.*/lib%s.so$" % req.library), + posixpath.join("/usr/lib/lib%s.a$" % req.library), + posixpath.join("/usr/lib/.*/lib%s.a$" % req.library), + ] + return apt_mgr.get_package_for_paths(paths, regex=True) + + +def resolve_ruby_file_req(apt_mgr, req): + paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] + package = apt_mgr.get_package_for_paths(paths) + if package is not None: + return package + paths = [ + posixpath.join( + r"/usr/share/rubygems-integration/all/gems/([^/]+)/" + "lib/%s.rb" % req.filename + ) + ] + return apt_mgr.get_package_for_paths(paths, regex=True) + + +def resolve_xml_entity_req(apt_mgr, req): + # Ideally we should be using the XML catalog for this, but hardcoding + # a few URLs will do for now.. + URL_MAP = { + "http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/" + } + for url, path in URL_MAP.items(): + if req.url.startswith(url): + search_path = posixpath.join(path, req.url[len(url) :]) + break + else: + return None + + return apt_mgr.get_package_for_paths([search_path], regex=False) + + +def resolve_sprockets_file_req(apt_mgr, req): + if req.content_type == "application/javascript": + path = "/usr/share/.*/app/assets/javascripts/%s.js$" % req.name + else: + logging.warning("unable to handle content type %s", req.content_type) + return None + return apt_mgr.get_package_for_paths([path], regex=True) + + +def resolve_java_class_req(apt_mgr, req): + # Unfortunately this only finds classes in jars installed on the host + # system :( + # TODO(jelmer): Call in session + output = apt_mgr.session.check_output( + ["java-propose-classpath", "-c" + req.classname]) + classpath = [p for p in output.decode().strip(":").strip().split(":") if p] + if not classpath: + logging.warning("unable to find classpath for %s", req.classname) + return False + logging.info("Classpath for %s: %r", req.classname, classpath) + package = apt_mgr.get_package_for_paths(classpath) + if package is None: + logging.warning("no package for files in %r", classpath) + return None + return package + + +def resolve_haskell_package_req(apt_mgr, req): + path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0] + return apt_mgr.get_package_for_paths([path], regex=True) + + +def resolve_maven_artifact_req(apt_mgr, req): + artifact = req.artifacts[0] + parts = artifact.split(":") + if len(parts) == 4: + (group_id, artifact_id, kind, version) = parts + regex = False + elif len(parts) == 3: + (group_id, artifact_id, version) = parts + kind = "jar" + regex = False + elif len(parts) == 2: + version = ".*" + (group_id, artifact_id) = parts + kind = "jar" + regex = True + else: + raise AssertionError("invalid number of parts to artifact %s" % artifact) + paths = [ + posixpath.join( + "/usr/share/maven-repo", + group_id.replace(".", "/"), + artifact_id, + version, + "%s-%s.%s" % (artifact_id, version, kind), + ) + ] + return apt_mgr.get_package_for_paths(paths, regex=regex) + + +def resolve_gnome_common_req(apt_mgr, req): + return 'gnome-common' + + +def resolve_jdk_file_req(apt_mgr, req): + path = req.jdk_path + ".*/" + req.filename + return apt_mgr.get_package_for_paths([path], regex=True) + + APT_REQUIREMENT_RESOLVERS = [ (BinaryRequirement, resolve_binary_req), + (PkgConfigRequirement, resolve_pkg_config_req), + (PathRequirement, resolve_path_req), + (CHeaderRequirement, resolve_c_header_req), + (JavaScriptRuntimeRequirement, resolve_js_runtime_req), + (ValaPackageRequirement, resolve_vala_package_req), + (RubyGemRequirement, resolve_ruby_gem_req), + (GoPackageRequirement, resolve_go_package_req), + (DhAddonRequirement, resolve_dh_addon_req), + (PhpClassRequirement, resolve_php_class_req), + (RPackageRequirement, resolve_r_package_req), + (NodePackageRequirement, resolve_node_package_req), + (LibraryRequirement, resolve_library_req), + (RubyFileRequirement, resolve_ruby_file_req), + (XmlEntityRequirement, resolve_xml_entity_req), + (SprocketsFileRequirement, resolve_sprockets_file_req), + (JavaClassRequirement, resolve_java_class_req), + (HaskellPackageRequirement, resolve_haskell_package_req), + (MavenArtifactRequirement, resolve_maven_artifact_req), + (GnomeCommonRequirement, resolve_gnome_common_req), + (JDKFileRequirement, resolve_jdk_file_req), ] @@ -70,14 +358,12 @@ class AptResolver(Resolver): def explain(self, requirements): raise NotImplementedError(self.explain) - def resolve(self, requirements): - for req in requirements: - for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: - if isinstance(req, rr_class): - package_name = rr_fn(self.apt, req) - if package_name is None: - raise NoAptPackage() - yield package_name - break - else: - raise NotImplementedError + def resolve(self, req: UpstreamRequirement): + for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: + if isinstance(req, rr_class): + deb_req = rr_fn(self.apt, req) + if deb_req is None: + raise NoAptPackage() + return deb_req + else: + raise NotImplementedError